diff --git a/.github/workflows/build_wheels_and_releases.yml b/.github/workflows/build_wheels_and_releases.yml new file mode 100644 index 0000000000..b6801e26d8 --- /dev/null +++ b/.github/workflows/build_wheels_and_releases.yml @@ -0,0 +1,187 @@ +name: Build-Wheels-PyPi +# https://github.com/pypa/cibuildwheel +# Controls when the workflow will run +on: + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Build the wheels for Linux, Windows and macOS for Python 3.8 and newer + build_wheels: + name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: python + + strategy: + # Ensure that a wheel builder finishes even if another fails + fail-fast: false + matrix: + include: + # Window 64 bit + - os: windows-2019 + python: 38 + bitness: 64 + platform_id: win_amd64 + - os: windows-latest + python: 39 + bitness: 64 + platform_id: win_amd64 + - os: windows-latest + python: 310 + bitness: 64 + platform_id: win_amd64 + + # Window 32 bit + - os: windows-latest + python: 38 + bitness: 32 + platform_id: win32 + - os: windows-latest + python: 39 + bitness: 32 + platform_id: win32 + + # Linux 64 bit manylinux2014 + - os: ubuntu-latest + python: 38 + bitness: 64 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python: 39 + bitness: 64 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + + # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014 + - os: ubuntu-latest + python: 310 + bitness: 64 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + + # MacOS x86_64 + - os: macos-latest + bitness: 64 + python: 38 + platform_id: macosx_x86_64 + - os: macos-latest + bitness: 64 + python: 39 + platform_id: macosx_x86_64 + - os: macos-latest + bitness: 64 + python: 310 + platform_id: macosx_x86_64 + + # MacOS arm64 + - os: macos-latest + bitness: 64 + python: 38 + platform_id: macosx_arm64 + - os: macos-latest + bitness: 64 + python: 39 + platform_id: macosx_arm64 + - os: macos-latest + bitness: 64 + python: 310 + platform_id: macosx_arm64 + + steps: + - name: Checkout fedml + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + #with: + # python-version: '3.9' + + - name: Build and test wheels + env: + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_ARCHS: all + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_TEST_SKIP: "*-macosx_arm64" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_BUILD_VERBOSITY: 1 + + #run: bash build_tools/github/build_wheels.sh + run: | + python -m pip install -U wheel setuptools + python setup.py sdist bdist_wheel + ls dist/*.whl + + - name: Store artifacts + uses: actions/upload-artifact@v3 + with: + path: python/dist/*.whl + + # Build the source distribution under Linux + build_sdist: + name: Source distribution + needs: [ build_wheels ] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: python + + steps: + - name: Checkout fedml + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' # update once build dependencies are available + + - name: Build source distribution + run: bash build_tools/github/build_source.sh + + - name: Test source distribution + run: bash build_tools/github/test_source.sh + env: + OMP_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 + + - name: Store artifacts + uses: actions/upload-artifact@v3 + with: + path: python/dist/*.tar.gz + + upload_pypi: + name: Upload pypi + needs: [ build_sdist ] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: python + # upload to PyPI on every tag starting with 'v' + # if: github.event_name == 'push' && contains(github.event.comment, 'release v') + # alternatively, to publish when a GitHub Release is created, use the following rule: + # if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: python/dist + + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + skip_existing: true + packages_dir: python/dist + user: ${{ secrets.PYPI_USER_NAME }} + password: ${{ secrets.PYPI_PASSWORD }} + # To test: repository_url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/build_wheels_and_releases.yml-backup b/.github/workflows/build_wheels_and_releases.yml-backup new file mode 100644 index 0000000000..2a27b44e65 --- /dev/null +++ b/.github/workflows/build_wheels_and_releases.yml-backup @@ -0,0 +1,250 @@ +#name: Build_Wheels_and_Release +## https://github.com/pypa/cibuildwheel +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the master branch +# schedule: +# # Nightly build at 12:12 A.M. +# - cron: "12 12 */1 * *" +## push: +## branches: [ master, test/v0.7.0 ] +## pull_request: +## branches: [ master, test/v0.7.0 ] +# +# # Allows you to run this workflow manually from the Actions tab +# workflow_dispatch: +# +#jobs: +# build_wheels: +# runs-on: [self-hosted, devops] +# defaults: +# run: +# shell: bash +# working-directory: python +# strategy: +# # Ensure that a wheel builder finishes even if another fails +# fail-fast: false +# matrix: +# # Github Actions doesn't support pairing matrix values together, let's improvise +# # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 +# buildplat: +# - [ubuntu-20.04, manylinux_x86_64] +# - [macos-10.15, macosx_*] +# - [windows-2019, win_amd64] +# - [windows-2019, win32] +# # TODO: uncomment PyPy 3.9 builds once PyPy +# # re-releases a new minor version +# # NOTE: This needs a bump of cibuildwheel version, also, once that happens. +# python: ["cp38", "cp39", "cp310", "pp38"] #, "pp39"] +# exclude: +# # Don't build PyPy 32-bit windows +# - buildplat: [windows-2019, win32] +# python: "pp38" +# - buildplat: [windows-2019, win32] +# python: "pp39" +# env: +# IS_32_BIT: ${{ matrix.buildplat[1] == 'win32' }} +# IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} +# IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} +# steps: +# - name: Checkout numpy +# - uses: actions/checkout@v3 +# +# # Used to push the built wheels +# - uses: actions/setup-python@v3 +# with: +# python-version: '3.x' +# +# - name: Configure mingw for 32-bit builds +# run: | +# # Force 32-bit mingw +# choco uninstall mingw +# choco install -y mingw --forcex86 --force --version=7.3.0 +# echo "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw32\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append +# refreshenv +# if: ${{ env.IS_32_BIT == 'true' }} +# +# +# - name: Install cibuildwheel +# run: | +# pip install --upgrade setuptools +# python3 -m pip install pipx +# python3 -m pipx ensurepath +# export PATH="${PATH}:$(python3 -c 'import site; print(site.USER_BASE)')/bin" +## python3 -m pipx ensurepath --force +# +# - name: Build wheels +# uses: pypa/cibuildwheel@v2.7.0 +# env: +# CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }} +# +# - uses: actions/upload-artifact@v3 +# with: +# name: ${{ matrix.python }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }} +# path: ./wheelhouse/*.whl + +# - name: Upload wheels +# if: success() +# shell: bash +# env: +# NUMPY_STAGING_UPLOAD_TOKEN: ${{ secrets.NUMPY_STAGING_UPLOAD_TOKEN }} +# NUMPY_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.NUMPY_NIGHTLY_UPLOAD_TOKEN }} +# run: | +# source tools/wheels/upload_wheels.sh +# set_upload_vars +# # trigger an upload to +# # https://anaconda.org/scipy-wheels-nightly/numpy +# # for cron jobs or "Run workflow" (restricted to main branch). +# # Tags will upload to +# # https://anaconda.org/multibuild-wheels-staging/numpy +# # The tokens were originally generated at anaconda.org +# upload_wheels +# strategy: +# fail-fast: false +# matrix: +# os: [ ubuntu-20.04, windows-2019, macOS-10.15 ] +# arch: [X86, X64, ARM, ARM64] +# python-version: ['3.6', '3.7', '3.8', '3.9'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' +# strategy: +# # Ensure that a wheel builder finishes even if another fails +# fail-fast: false +# matrix: +# include: +# # from mpi4py import MPI +# # ImportError: DLL load failed while importing MPI: The specified module could not be found. +# - os: windows-2019-py38-amd64 +# python: 38 +# bitness: 64 +# platform_id: win_amd64 +# # from mpi4py import MPI +# # ImportError: DLL load failed while importing MPI: The specified module could not be found. +# - os: windows-latest-py39-amd64 +# python: 39 +# bitness: 64 +# platform_id: win_amd64 +# # ERROR: No matching distribution found for MNN==1.1.6 +# - os: windows-latest-py310-amd64 +# python: 310 +# bitness: 64 +# platform_id: win_amd64 +# # #RROR: No matching distribution found for torch==1.11.0 +# - os: windows-latest-py38-win32 +# python: 38 +# bitness: 32 +# platform_id: win32 +# # ERROR: No matching distribution found for torch==1.11.0 +# - os: windows-latest-py39-win32 +# python: 39 +# bitness: 32 +# platform_id: win32 +# +# # auditwheel.main_repair:This does not look like a platform wheel +# - os: ubuntu-latest-py38-x86 +# python: 38 +# bitness: 64 +# platform_id: manylinux_x86_64 +# manylinux_image: manylinux2014 +# # auditwheel.main_repair:This does not look like a platform wheel +# - os: ubuntu-latest-py39-x86 +# python: 39 +# bitness: 64 +# platform_id: manylinux_x86_64 +# manylinux_image: manylinux2014 +# # auditwheel.main_repair:This does not look like a platform wheel +# - os: ubuntu-latest-py310-x86 +# python: 310 +# bitness: 64 +# platform_id: manylinux_x86_64 +# manylinux_image: manylinux2014 +# +# # _configtest.c:2:10: fatal error: 'mpi.h' file not found +# - os: macos-latest-py38-x86 +# bitness: 64 +# python: 38 +# platform_id: macosx_x86_64 +# # _configtest.c:2:10: fatal error: 'mpi.h' file not found +# - os: macos-latest-py39-x86 +# bitness: 64 +# python: 39 +# platform_id: macosx_x86_64 +# # _configtest.c:2:10: fatal error: 'mpi.h' file not found +# - os: macos-latest-py310-x86 +# bitness: 64 +# python: 310 +# platform_id: macosx_x86_64 +# +# # MacOS arm64 +# - os: macos-latest-py38-arm64 +# bitness: 64 +# python: 38 +# platform_id: macosx_arm64 +# - os: macos-latest-py39-arm64 +# bitness: 64 +# python: 39 +# platform_id: macosx_arm64 +# - os: macos-latest-py310-arm64 +# bitness: 64 +# python: 310 +# platform_id: macosx_arm64 + +# steps: +# - uses: actions/checkout@v3 +# +# # Used to host cibuildwheel +# - uses: actions/setup-python@v3 +# +## - name: Install cibuildwheel +## run: python -m pip install cibuildwheel==2.7.0 +# +# - name: Build wheels +# working-directory: ./python +# env: +# CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} +# CIBW_ARCHS: all +# CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash {project}/python/build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} +# CIBW_BEFORE_TEST_WINDOWS: bash {project}/python/build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} +# CIBW_TEST_COMMAND: bash {project}/python/build_tools/github/test_wheels.sh +# CIBW_TEST_COMMAND_WINDOWS: bash {project}/python/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} +# CIBW_BUILD_VERBOSITY: 1 +# run: | +# python -m pip install -U wheel setuptools +# python setup.py sdist bdist_wheel +# run: python -m cibuildwheel --output-dir dist +## run: cd {project}/python && python -m cibuildwheel --output-dir wheelhouse +# # to supply options, put them in 'env', like (test) +# # env: +# # CIBW_SOME_OPTION: value +# +# - name: Upload source zip file +# uses: actions/upload-artifact@v2 +# with: +# path: python/dist/*.tar.gz +# - name: Upload Wheels +# uses: actions/upload-artifact@v2 +# with: +# path: ./python/dist/*.whl + +# upload_pypi: +# needs: [build_wheels] +# runs-on: [self-hosted] +# # upload to PyPI on every tag starting with 'v' +## if: github.event_name == 'push' && contains(github.event.comment, 'release v') +# # alternatively, to publish when a GitHub Release is created, use the following rule: +# # if: github.event_name == 'release' && github.event.action == 'published' +# steps: +# - uses: actions/download-artifact@v2 +# with: +# name: artifact +# path: dist +# +# - uses: pypa/gh-action-pypi-publish@v1.4.2 +# with: +# skip_existing: true +# user: chaoyanghe +# password: ${{ secrets.pypi_password }} +# # To test: repository_url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/full_e2e_test.yml-bakcup b/.github/workflows/full_e2e_test.yml-bakcup new file mode 100644 index 0000000000..4dd408fbd8 --- /dev/null +++ b/.github/workflows/full_e2e_test.yml-bakcup @@ -0,0 +1,104 @@ +# This is a basic workflow to help you get started with Actions + +name: Full End-to-end Test + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master, test/v0.7.0 ] + pull_request: + branches: [ master, test/v0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "mlops-cli-test" + mlops-cli-test: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + arch: [X64, ARM64] + python-version: ['3.8'] + # The type of runner that the job will run on + runs-on: [self-hosted, devops] + defaults: + run: + shell: bash + working-directory: python + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + # - name: pip install -e ./ + # run: | + # echo "install pip" + # sudo apt-get install python3-pip << eof + # + # y + # + # eof + # + # echo "pip install -e ./" + # pip install -e ./ + + - name: test sp_fedavg_mnist_lr_example + run: | + echo "this is for test sp_fedavg_mnist_lr_example" + cd examples/simulation/sp_fedavg_mnist_lr_example + + python torch_fedavg_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_fedopt_mnist_lr_example + run: | + echo "this is for test sp_fedopt_mnist_lr_example" + cd examples/simulation/sp_fedopt_mnist_lr_example + + python torch_fedopt_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_fednova_mnist_lr_example + run: | + echo "this is for test sp_fednova_mnist_lr_example" + cd examples/simulation/sp_fednova_mnist_lr_example + + python torch_fednova_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_turboaggregate_mnist_lr_example + run: | + echo "this is for test sp_turboaggregate_mnist_lr_example" + cd examples/simulation/sp_turboaggregate_mnist_lr_example + + python torch_turboaggregate_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_hierarchicalfl_mnist_lr_example + run: | + echo "this is for test sp_hierarchicalfl_mnist_lr_example" + cd examples/simulation/sp_hierarchicalfl_mnist_lr_example + + python torch_hierarchicalfl_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_vertical_mnist_lr_example + run: | + echo "this is for test sp_vertical_mnist_lr_example" + cd examples/simulation/sp_vertical_mnist_lr_example + + python torch_vertical_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp_fedsgd_cifar10_resnet20_example + run: | + echo "this is for test sp_fedsgd_cifar10_resnet20_example" + + cd examples/simulation/sp_fedsgd_cifar10_resnet20_example + + python sp_fedsgd_cifar10_resnet20_example.py --cf eftopk_config.yaml + + - name: test example B + run: | + echo "this is for test example B" + echo "second line of the script" \ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index a4f4bf6f58..ae7fc77858 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,15 +1,34 @@ -name: Pylint +name: Pylint - FedML -on: [push] +on: + push: + branches: [ master, test/v0.7.0, dev/v0.7.0 ] + + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: build: - runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: python + runs-on: [self-hosted, runner-linux, devops-mpi] strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8"] + mpi: [mpich] +# include: +# - os: ubuntu-latest +# mpi: mpich +# install-mpi: sudo apt install -y mpich libmpich-dev steps: - uses: actions/checkout@v3 +# - name: Install MPI +# run: ${{ matrix.install-mpi }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: @@ -20,4 +39,13 @@ jobs: pip install pylint - name: Analysing the code with pylint run: | - pylint $(git ls-files '*.py') + python -m pip install mpi4py + pip install "fedml[gRPC]" + pip install "fedml[tensorflow]" + pip install "fedml[jax]" + pip install "fedml[mxnet]" + pip install tensorflow_federated + pip install mxnet + pip install jax + pip install ptflops + pylint --rcfile=build_tools/lint/.pylintrc --disable=C,R,W,I ./ diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index ec703542be..0000000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/runner.md b/.github/workflows/runner.md new file mode 100644 index 0000000000..40b5462a4d --- /dev/null +++ b/.github/workflows/runner.md @@ -0,0 +1,37 @@ +# Install GitHub runner with your own computer: +ssh -i amir-github-actions-key.cer ubuntu@54.183.200.162 +ssh -i amir-github-actions-key.cer ubuntu@52.53.164.162 +ssh -i github_actions.cer ubuntu@54.153.18.24 + +sudo rpm -Uvh https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm +sudo apt-get update && sudo apt-get install -y dotnet6 +dotnet --version +#install runner based on the following url: https://github.com/FedML-AI/FedML/settings/actions/runners/new?arch=x64&os=linux +sudo ./svc.sh install +sudo ./svc.sh start +sudo ./svc.sh status + +# Install GitHub runner in Ubuntu from AWS: +ssh -i "fedml-github-action.pem" ubuntu@ec2-54-176-61-229.us-west-1.compute.amazonaws.com +ssh -i "fedml-github-action.pem" ubuntu@ec2-54-219-186-81.us-west-1.compute.amazonaws.com +ssh -i "fedml-github-action.pem" ubuntu@ec2-54-219-187-134.us-west-1.compute.amazonaws.com + +sudo rpm -Uvh https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm +sudo apt-get update && sudo apt-get install -y dotnet6 +dotnet --version +#install runner based on the following url: https://github.com/FedML-AI/FedML/settings/actions/runners/new?arch=x64&os=linux + +sudo ./svc.sh install +sudo ./svc.sh start +sudo ./svc.sh status + + +# Install GitHub runner in Windows from AWS: +1. You may connect to AWS Windows server by RDP client from MAC AppStore based on the url: https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-mac + +host: ec2-184-169-242-201.us-west-1.compute.amazonaws.com + +2. Enabling Windows Long Path on Windows based on the following url: + https://www.microfocus.com/documentation/filr/filr-4/filr-desktop/t47bx2ogpfz7.html + +3. install runner based on the following url: https://github.com/FedML-AI/FedML/settings/actions/runners/new?arch=x64&os=win diff --git a/.github/workflows/smoke_test_cross_device_mnn_server_linux.yml b/.github/workflows/smoke_test_cross_device_mnn_server_linux.yml new file mode 100644 index 0000000000..b2fea66602 --- /dev/null +++ b/.github/workflows/smoke_test_cross_device_mnn_server_linux.yml @@ -0,0 +1,51 @@ +# This is a basic workflow to help you get started with Actions + +name: CROSS-DEVICE-MNN-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-device-mnn-server: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + arch: [X64] + python-version: ['3.8'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: test server of cross-device + run: | + cd quick_start/beehive + timeout 60 bash run_server.sh || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + diff --git a/.github/workflows/smoke_test_cross_silo_fedavg_attack_linux.yml b/.github/workflows/smoke_test_cross_silo_fedavg_attack_linux.yml new file mode 100644 index 0000000000..aa6cdf4b8d --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_fedavg_attack_linux.yml @@ -0,0 +1,85 @@ +# This is a basic workflow to help you get started with Actions + +name: Attacker-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-attack-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2', '3', '4'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - attack + run: | + cd examples/cross_silo/mqtt_s3_fedavg_attack_mnist_lr_example + run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - attack + run: | + cd examples/cross_silo/mqtt_s3_fedavg_attack_mnist_lr_example + run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - attack + run: | + cd examples/cross_silo/mqtt_s3_fedavg_attack_mnist_lr_example + run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} + + - name: client 3 - cross-silo - attack + run: | + cd examples/cross_silo/mqtt_s3_fedavg_attack_mnist_lr_example + run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 3 $run_id + if: ${{ matrix.client-index == '3' }} + + - name: client 4 - cross-silo - attack + run: | + cd examples/cross_silo/mqtt_s3_fedavg_attack_mnist_lr_example + run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 4 $run_id + if: ${{ matrix.client-index == '4' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_fedavg_cdp_linux.yml b/.github/workflows/smoke_test_cross_silo_fedavg_cdp_linux.yml new file mode 100644 index 0000000000..2272285811 --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_fedavg_cdp_linux.yml @@ -0,0 +1,69 @@ +# This is a basic workflow to help you get started with Actions + +name: CDP-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-cdp-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - cdp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_cdp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - cdp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_cdp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - cdp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_cdp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_fedavg_defense_linux.yml b/.github/workflows/smoke_test_cross_silo_fedavg_defense_linux.yml new file mode 100644 index 0000000000..e009288ff5 --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_fedavg_defense_linux.yml @@ -0,0 +1,85 @@ +# This is a basic workflow to help you get started with Actions + +name: Defender-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-defense-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2', '3', '4'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - defense + run: | + cd examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example + run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - defense + run: | + cd examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example + run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - defense + run: | + cd examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example + run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} + + - name: client 3 - cross-silo - defense + run: | + cd examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example + run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 3 $run_id + if: ${{ matrix.client-index == '3' }} + + - name: client 4 - cross-silo - defense + run: | + cd examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example + run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 4 $run_id + if: ${{ matrix.client-index == '4' }} diff --git a/.github/workflows/smoke_test_cross_silo_fedavg_ldp_linux.yml b/.github/workflows/smoke_test_cross_silo_fedavg_ldp_linux.yml new file mode 100644 index 0000000000..9fb43f6bfb --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_fedavg_ldp_linux.yml @@ -0,0 +1,69 @@ +# This is a basic workflow to help you get started with Actions + +name: LDP-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-ldp-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - ldp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_ldp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - ldp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_ldp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - ldp + run: | + cd examples/cross_silo/mqtt_s3_fedavg_ldp_mnist_lr_example + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_ho_linux.yml b/.github/workflows/smoke_test_cross_silo_ho_linux.yml new file mode 100644 index 0000000000..09497f830c --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_ho_linux.yml @@ -0,0 +1,69 @@ +# This is a basic workflow to help you get started with Actions + +name: CROSS-SILO-HO-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-horizontal-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - ho + run: | + cd quick_start/octopus + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - ho + run: | + cd quick_start/octopus + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - ho + run: | + cd quick_start/octopus + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_ho_win.yml b/.github/workflows/smoke_test_cross_silo_ho_win.yml new file mode 100644 index 0000000000..a1b57d7bd2 --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_ho_win.yml @@ -0,0 +1,70 @@ +# This is a basic workflow to help you get started with Actions + +name: CROSS-SILO-HO-Win + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-horizontal-test: + defaults: + run: + shell: powershell + working-directory: fedml-devops\python + strategy: + fail-fast: false + matrix: + os: [ windows-2019 ] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-windows, devops] + timeout-minutes: 15 + steps: + - name: cleanup running processes + continue-on-error: true + run: | + wmic.exe /interactive:off process where "name='python.exe'" call terminate + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + with: + path: fedml-devops + clean: true + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - ho + run: | + cd quick_start/octopus + .\run_server.bat ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - ho + run: | + cd quick_start/octopus + .\run_client.bat 1 ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - ho + run: | + cd quick_start/octopus + .\run_client.bat 2 ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_lightsecagg_linux.yml b/.github/workflows/smoke_test_cross_silo_lightsecagg_linux.yml new file mode 100644 index 0000000000..f408760742 --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_lightsecagg_linux.yml @@ -0,0 +1,69 @@ +# This is a basic workflow to help you get started with Actions + +name: LightSecAgg-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-horizontal-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - lightsecagg + run: | + cd examples/cross_silo/light_sec_agg_example + run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - lightsecagg + run: | + cd examples/cross_silo/light_sec_agg_example + run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - lightsecagg + run: | + cd examples/cross_silo/light_sec_agg_example + run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_cross_silo_lightsecagg_win.yml b/.github/workflows/smoke_test_cross_silo_lightsecagg_win.yml new file mode 100644 index 0000000000..825c67f2b4 --- /dev/null +++ b/.github/workflows/smoke_test_cross_silo_lightsecagg_win.yml @@ -0,0 +1,70 @@ +# This is a basic workflow to help you get started with Actions + +name: LightSecAgg-Windows + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + cross-silo-horizontal-test: + defaults: + run: + shell: powershell + working-directory: fedml-devops\python + strategy: + fail-fast: false + matrix: + os: [ windows-2019 ] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-windows, devops] + timeout-minutes: 15 + steps: + - name: cleanup running processes + continue-on-error: true + run: | + wmic.exe /interactive:off process where "name='python.exe'" call terminate + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + with: + path: fedml-devops + clean: true + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - cross-silo - ho + run: | + cd examples/cross_silo/light_sec_agg_example + .\run_server.bat cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - cross-silo - ho + run: | + cd examples/cross_silo/light_sec_agg_example + .\run_client.bat 1 cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - cross-silo - lightsecagg + run: | + cd examples/cross_silo/light_sec_agg_example + .\run_client.bat 2 cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_flow_linux.yml b/.github/workflows/smoke_test_flow_linux.yml new file mode 100644 index 0000000000..6a1b2a3ed5 --- /dev/null +++ b/.github/workflows/smoke_test_flow_linux.yml @@ -0,0 +1,69 @@ +# This is a basic workflow to help you get started with Actions + +name: Flow-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + Flow-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: server - Flow + run: | + cd fedml/core/distributed/flow + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash test_run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - Flow + run: | + cd fedml/core/distributed/flow + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash test_run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - Flow + run: | + cd fedml/core/distributed/flow + run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash test_run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} \ No newline at end of file diff --git a/.github/workflows/smoke_test_ml_engines_linux.yml b/.github/workflows/smoke_test_ml_engines_linux.yml new file mode 100644 index 0000000000..e90c438739 --- /dev/null +++ b/.github/workflows/smoke_test_ml_engines_linux.yml @@ -0,0 +1,176 @@ +# This is a basic workflow to help you get started with Actions + +name: ML-Engines-Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + tf-ml-engines-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] + client-index: ['0', '1', '2'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + pip install -e '.[tensorflow]' + + - name: server - tensorflow - fedavg + run: | + cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example + run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - tensorflow - fedavg + run: | + cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example + run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - tensorflow - fedavg + run: | + cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example + run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} + + jax-ml-engines-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + arch: [ X64 ] + python-version: [ '3.8' ] + client-index: [ '0', '1', '2' ] + # exclude: + # - os: macos-latest + # python-version: '3.8' + # - os: windows-latest + # python-version: '3.6' + runs-on: [ self-hosted, runner-linux, devops ] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + pip install -e '.[jax]' + + - name: server - jax - fedavg + run: | + cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example + run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - jax - fedavg + run: | + cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example + run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - jax - fedavg + run: | + cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example + run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} + + mxnet-ml-engines-test: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + arch: [ X64 ] + python-version: [ '3.8' ] + client-index: [ '0', '1', '2' ] + # exclude: + # - os: macos-latest + # python-version: '3.8' + # - os: windows-latest + # python-version: '3.6' + runs-on: [ self-hosted, runner-linux, devops ] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + pip install -e '.[mxnet]' + + - name: server - mxnet - fedavg + run: | + cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example + run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_server.sh $run_id + if: ${{ matrix.client-index == '0' }} + + - name: client 1 - mxnet - fedavg + run: | + cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example + run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 1 $run_id + if: ${{ matrix.client-index == '1' }} + + - name: client 2 - mxnet - fedavg + run: | + cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example + run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }} + echo ${run_id} + bash run_client.sh 2 $run_id + if: ${{ matrix.client-index == '2' }} diff --git a/.github/workflows/smoke_test_pip_cli_sp_linux.yml b/.github/workflows/smoke_test_pip_cli_sp_linux.yml new file mode 100644 index 0000000000..d5381ccb4c --- /dev/null +++ b/.github/workflows/smoke_test_pip_cli_sp_linux.yml @@ -0,0 +1,88 @@ +# This is a basic workflow to help you get started with Actions + +name: PIP, CLI, SP - On Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +permissions: write-all + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + pip-install-fedml-and-test-sp: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + # https://github.com/actions/checkout/issues/116#issuecomment-644419389 + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: test "fedml login" and "fedml build" + run: | + cd tests/smoke_test/cli + bash login.sh + bash build.sh + - name: test simulation-sp + run: | + cd quick_start/parrot + python torch_fedavg_mnist_lr_one_line_example.py --cf fedml_config.yaml + python torch_fedavg_mnist_lr_custum_data_and_model_example.py --cf fedml_config.yaml + + - name: test sp - sp_decentralized_mnist_lr_example + run: | + cd examples/simulation/sp_decentralized_mnist_lr_example + python torch_fedavg_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp - sp_fednova_mnist_lr_example + run: | + cd examples/simulation/sp_fednova_mnist_lr_example + python torch_fednova_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp - sp_fedopt_mnist_lr_example + run: | + cd examples/simulation/sp_fedopt_mnist_lr_example + python torch_fedopt_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp - sp_hierarchicalfl_mnist_lr_example + run: | + cd examples/simulation/sp_hierarchicalfl_mnist_lr_example + python torch_hierarchicalfl_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp - sp_turboaggregate_mnist_lr_example + run: | + cd examples/simulation/sp_turboaggregate_mnist_lr_example + python torch_turboaggregate_mnist_lr_step_by_step_example.py --cf fedml_config.yaml + + - name: test sp - sp_vertical_mnist_lr_example + run: | + cd examples/simulation/sp_vertical_mnist_lr_example + python torch_vertical_mnist_lr_step_by_step_example.py --cf fedml_config.yaml diff --git a/.github/workflows/smoke_test_pip_cli_sp_win.yml b/.github/workflows/smoke_test_pip_cli_sp_win.yml new file mode 100644 index 0000000000..e60546122d --- /dev/null +++ b/.github/workflows/smoke_test_pip_cli_sp_win.yml @@ -0,0 +1,63 @@ +# This is a basic workflow to help you get started with Actions + +name: PIP, CLI, SP - On Windows + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + pip-install-fedml-and-test-sp: + defaults: + run: + shell: powershell + working-directory: fedml-devops\python + strategy: + fail-fast: false + matrix: + os: [ windows-2019 ] + arch: [X64] + python-version: ['3.8'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-windows, devops] + timeout-minutes: 15 + steps: + - name: cleanup running processes + continue-on-error: true + run: | + wmic.exe /interactive:off process where "name='python.exe'" call terminate + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + # https://github.com/actions/checkout/issues/116#issuecomment-644419389 + - uses: actions/checkout@v3 + with: + path: fedml-devops + clean: true + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: test "fedml login" and "fedml build" + run: | + cd tests/smoke_test/cli + .\login.bat + .\build.bat + - name: test simulation-sp + run: | + cd quick_start/parrot + python torch_fedavg_mnist_lr_one_line_example.py --cf fedml_config.yaml + python torch_fedavg_mnist_lr_custum_data_and_model_example.py --cf fedml_config.yaml diff --git a/.github/workflows/smoke_test_security.yml b/.github/workflows/smoke_test_security.yml new file mode 100644 index 0000000000..2911809b26 --- /dev/null +++ b/.github/workflows/smoke_test_security.yml @@ -0,0 +1,57 @@ +# This is a basic workflow to help you get started with Actions + +name: Security(attack/defense) on Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +permissions: write-all + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + security-attack-defense-tests: + defaults: + run: + shell: bash + working-directory: python + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest] + arch: [X64] + python-version: ['3.8'] +# exclude: +# - os: macos-latest +# python-version: '3.8' +# - os: windows-latest +# python-version: '3.6' + runs-on: [self-hosted, runner-linux, devops] + timeout-minutes: 15 + steps: + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + # https://github.com/actions/checkout/issues/116#issuecomment-644419389 + - uses: actions/checkout@v3 + - name: pip install -e ./ + run: | + pip install -e ./ + + - name: attack tests + run: | + cd tests/security + sh run_attacker_tests.sh + + - name: defense tests + run: | + cd tests/security + sh run_defender_tests.sh \ No newline at end of file diff --git a/.github/workflows/smoke_test_simulation_mpi_linux.yml b/.github/workflows/smoke_test_simulation_mpi_linux.yml new file mode 100644 index 0000000000..375398fcaf --- /dev/null +++ b/.github/workflows/smoke_test_simulation_mpi_linux.yml @@ -0,0 +1,94 @@ +# This is a basic workflow to help you get started with Actions + +name: MPI - On Linux + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + schedule: + # Nightly build at 12:12 A.M. + - cron: "12 12 */1 * *" + pull_request: + branches: [ master, test/v0.7.0, dev/0.7.0 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel + +jobs: + # https://github.com/mpi4py/mpi4py/actions/runs/34979774/workflow + mpi_run: + runs-on: [self-hosted, runner-linux, devops-mpi] + timeout-minutes: 15 + defaults: + run: + shell: bash + working-directory: python + strategy: + matrix: + python-version: [3.8] + mpi: [mpich] +# mpi: [mpich, openmpi] + os: [ ubuntu-latest ] +# include: +# - os: ubuntu-latest +# mpi: mpich +# install-mpi: sudo apt install -y mpich libmpich-dev +# - os: ubuntu-latest +# mpi: openmpi +# install-mpi: sudo apt install -y openmpi-bin libopenmpi-dev + steps: + - uses: actions/checkout@v2 + - name: Install MPI + run: ${{ matrix.install-mpi }} + - name: Use Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install packaging tools + run: python -m pip install --upgrade setuptools pip wheel + - name: Install build dependencies + run: python -m pip install --upgrade cython + - name: Build package + run: python -m pip wheel -vvv --wheel-dir=dist . + - name: Install test dependencies + run: python -m pip install --upgrade numpy + + - name: pip install -e ./ + run: | + pip install -e ./ + python -m pip install mpi4py + + - name: Test package - FedAvg + run: | + cd examples/simulation/mpi_torch_fedavg_mnist_lr_example + sh run_custom_data_and_model_example.sh 4 + + - name: Test package - Base + run: | + cd examples/simulation/mpi_base_framework_example + sh run.sh 4 + + - name: Test package - Decentralized + run: | + cd examples/simulation/mpi_decentralized_fl_example + sh run.sh 4 + + - name: Test package - FedOPT + run: | + cd examples/simulation/mpi_fedopt_datasets_and_models_example + sh run_step_by_step_example.sh 4 config/mnist_lr/fedml_config.yaml + + - name: Test package - FedProx + run: | + cd examples/simulation/mpi_fedprox_datasets_and_models_example + sh run_step_by_step_example.sh 4 config/mnist_lr/fedml_config.yaml + + - name: Test package - FedGAN + run: | + cd examples/simulation/mpi_torch_fedgan_mnist_gan_example + sh run_step_by_step_example.sh 4 + + - name: Uninstall package after testing + run: python -m pip uninstall --yes mpi4py \ No newline at end of file diff --git a/.gitignore b/.gitignore index ae1f7929c1..33bfde48a6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,14 @@ .idea .idea/* .vscode/* +*.pkl + +data/* +python/tests/smoke_test/data +python/tests/smoke_test/data/* + +android/fedmlsdk/MobileNN/MNN +android/fedmlsdk/MobileNN/pytorch *.pyc ./*/*.pyc @@ -10,6 +18,12 @@ ./*/*/*/*/*.pyc ./*/*/*/*/*/*.pyc +python/data +python/fedml.egg-info +python/examples/simulation/sp_fedavg_mnist_lr_example/data +python/examples/simulation/sp_fedsgd_cifar10_resnet20_example +python/examples/simulation/sp_fedsgd_cifar10_resnet20_example/* + *.log wandb @@ -24,7 +38,7 @@ python/dist python/FedML.egg-info doc/deploy -doc/en/_build +doc/en/_build/doctrees *.h5 @@ -37,6 +51,7 @@ cifar-10-batches-py *.zip *.json *.tar.bz2 +*.stats data/cifar10/*-python data/cifar100/*-python @@ -61,31 +76,106 @@ mimiconda.sh data/fednlp/* *.npz -test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.sh -test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.py -test/fedml_user_code/simulation_sp/data/mnist/MNIST/README.md -test/fedml_user_code/simulation_sp/data/mnist/MNIST/download_and_unzip.sh -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._train -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._test -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._.DS_Store -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.sh -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.py -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/README.md -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/download_and_unzip.sh -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._train -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._test -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._.DS_Store -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._.DS_Store -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._test -test/fedml_user_code/simulation_sp/data/mnist/__MACOSX/MNIST/._train -test/fedml_user_code/simulation_sp/data/mnist/MNIST/download_and_unzip.sh -test/fedml_user_code/simulation_sp/data/mnist/MNIST/README.md -test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.py -test/fedml_user_code/simulation_sp/data/mnist/MNIST/stats.sh -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._.DS_Store -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._test -test/fedml_user_code/simulation_mpi/data/mnist/__MACOSX/MNIST/._train -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/download_and_unzip.sh -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/README.md -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.py -test/fedml_user_code/simulation_mpi/data/mnist/MNIST/stats.sh +python/tests/smoke_test/simulation_sp/data/mnist/MNIST/stats.sh +python/tests/smoke_test/simulation_sp/data/mnist/MNIST/stats.py +python/tests/smoke_test/simulation_sp/data/mnist/MNIST/README.md +python/tests/smoke_test/simulation_sp/data/mnist/MNIST/download_and_unzip.sh +python/tests/smoke_test/simulation_sp/data/mnist/__MACOSX/MNIST/._train +python/tests/smoke_test/simulation_sp/data/mnist/__MACOSX/MNIST/._test +python/tests/smoke_test/simulation_sp/data/mnist/__MACOSX/MNIST/._.DS_Store +python/tests/smoke_test/simulation_mpi/data/mnist/MNIST/stats.sh +python/tests/smoke_test/simulation_mpi/data/mnist/MNIST/stats.py +python/tests/smoke_test/simulation_mpi/data/mnist/MNIST/README.md +python/tests/smoke_test/simulation_mpi/data/mnist/MNIST/download_and_unzip.sh +python/tests/smoke_test/simulation_mpi/data/mnist/__MACOSX/MNIST/._train +python/tests/smoke_test/simulation_mpi/data/mnist/__MACOSX/MNIST/._test +python/tests/smoke_test/simulation_mpi/data/mnist/__MACOSX/MNIST/._.DS_Store + +python/app/fednlp/text_classification/cache_dir +python/app/fednlp/seq2seq/cache_dir +python/app/fednlp/span_extraction/cache_dir + +app/fedgraphnn/moleculenet_graph_clf/data/bace/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/bace/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/bbbp/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/bbbp/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/clintox/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/clintox/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/hiv/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/hiv/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/lipo/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/lipo/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/muv/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/muv/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/pcba/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/pcba/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/sider/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/sider/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/tox21/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/tox21/*.npy +app/fedgraphnn/moleculenet_graph_clf/data/toxcast/*.pkl +app/fedgraphnn/moleculenet_graph_clf/data/toxcast/*.npy + +app/fedgraphnn/moleculenet_graph_reg/data/esol/*.pkl +app/fedgraphnn/moleculenet_graph_reg/data/esol/*.npy +app/fedgraphnn/moleculenet_graph_reg/data/freesolv/*.pkl +app/fedgraphnn/moleculenet_graph_reg/data/freesolv/*.npy +app/fedgraphnn/moleculenet_graph_reg/data/herg/*.pkl +app/fedgraphnn/moleculenet_graph_reg/data/herg/*.npy +app/fedgraphnn/moleculenet_graph_reg/data/lipo/*.pkl +app/fedgraphnn/moleculenet_graph_reg/data/lipo/*.npy +app/fedgraphnn/moleculenet_graph_reg/data/qm9/*.pkl +app/fedgraphnn/moleculenet_graph_reg/data/qm9/*.npy + + +python/app/fedgraphnn/social_networks_graph_clf/data/TUDataset/* +python/app/fedgraphnn/social_networks_graph_clf/data/TUDataset/collab.pickle +python/app/fedgraphnn/social_networks_graph_clf/x_hist.png + +python/app/fedgraphnn/social_networks_graph_clf/x_hist.png +python/app/healthcare/fed_lidc_idri /* + + + +python/app/fedgraphnn/social_networks_graph_clf/data/TUDataset/* +python/app/fedgraphnn/social_networks_graph_clf/data/TUDataset/collab.pickle +python/app/fedgraphnn/social_networks_graph_clf/x_hist.png + +app/fedgraphnn/subgraph_relation_pred/data/FB15k-237/*.dict +app/fedgraphnn/subgraph_relation_pred/data/FB15k-237/*.txt +app/fedgraphnn/subgraph_relation_pred/data/wn18rr/*.dict +app/fedgraphnn/subgraph_relation_pred/data/wn18rr/*.txt +app/fedgraphnn/subgraph_relation_pred/data/YAGO3-10/*.dict +app/fedgraphnn/subgraph_relation_pred/data/YAGO3-10/*.txt + +app/fedgraphnn/subgraph_link_pred/data/FB15k-237/*.dict +app/fedgraphnn/subgraph_link_pred/data/FB15k-237/*.txt +app/fedgraphnn/subgraph_link_pred/data/wn18rr/*.dict +app/fedgraphnn/subgraph_link_pred/data/wn18rr/*.txt +app/fedgraphnn/subgraph_link_pred/data/YAGO3-10/*.dict +app/fedgraphnn/subgraph_link_pred/data/YAGO3-10/*.txt + +python/app/fedgraphnn/ego_networks_link_pred/data/ego-networks +python/app/fedgraphnn/ego_networks_node_clf/data/ego-networks +/devops/scripts/aws/ +devops/scripts/docker +devops/scripts/kubectl +android/gradlew +android/gradlew.bat + +/iot/anomaly_detection_for_cybersecurity/data/Danmini_Doorbell/ +/iot/anomaly_detection_for_cybersecurity/data/Ecobee_Thermostat/ +/iot/anomaly_detection_for_cybersecurity/data/Ennio_Doorbell/ +/iot/anomaly_detection_for_cybersecurity/data/Philips_B120N10_Baby_Monitor/ +/iot/anomaly_detection_for_cybersecurity/data/Provision_PT_737E_Security_Camera/ +/iot/anomaly_detection_for_cybersecurity/data/Provision_PT_838_Security_Camera/ +/iot/anomaly_detection_for_cybersecurity/data/Samsung_SNH_1011_N_Webcam/ +/iot/anomaly_detection_for_cybersecurity/data/SimpleHome_XCS7_1002_WHT_Security_Camera/ +/iot/anomaly_detection_for_cybersecurity/data/SimpleHome_XCS7_1003_WHT_Security_Camera/ + + +doc/en/_build +/docker-2/ +/swap/actions-runner/ +*.txt +python/examples/cross_silo/light_sec_agg_example/mpi_host_file diff --git a/README.md b/README.md index 813509a91b..59b4b461e2 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,44 @@ -# FedML: The Community Connecting and Building AI Anywhere at Any Scale +# FedML: The Community Building Open and Collaborative AI Anywhere at Any Scale
- - +
At the current stage, FedML library provides a research and production integrated edge-cloud platform for Federated/Distributed Machine Learning at anywhere at any scale. -Homepage: [https://FedML.ai](https://FedML.ai) -

- Roadmap - Python3 - PyTorch - Travis - Contributors -

+FedML Homepage: https://fedml.ai/ \ +FedML Open Source: https://github.com/FedML-AI \ +FedML Platform: https://open.fedml.ai \ +FedML Use Cases: https://open.fedml.ai/platform/appStore \ +FedML Documentation: https://doc.fedml.ai \ +FedML Blog: https://medium.com/@FedML \ +FedML Research: https://fedml.ai/research-papers/ \ +FedML Product Overview: https://medium.com/@FedML/fedml-ai-platform-releases-the-worlds-federated-learning-open-platform-on-public-cloud-with-an-8024e68a70b6 + +Join the Community: \ +Slack: https://join.slack.com/t/fedml/shared_invite/zt-havwx1ee-a1xfOUrATNfc9DFqU~r34w \ +Discord: https://discord.gg/9xkW8ae6RV + +[//]: # () +[//]: # (

) + +[//]: # ( Roadmap) + +[//]: # ( Python3) + +[//]: # ( PyTorch) + +[//]: # ( Travis) + +[//]: # ( Contributors) + +[//]: # (

) # News +* [2022/08/01] (Product Introduction) FedML AI platform releases the world’s federated learning open platform on the public cloud with an in-depth introduction of products and technologies! Please visit this blog for details. + * [2022/03/15] (Fundraising): FedML, Inc. has finished the 1st-round fundraising. We are backed by top VCs who focus on AI, SaaS, and Blockchain/Web3/Crypto from the Bay Area, California of the USA. * [2022/02/14] (Company): FedML is upgraded as a Delaware-registered C-Corp company. Our headquarter is in California, USA. The two co-founders are CEO Salman Avestimehr (https://www.avestimehr.com/) and CTO Chaoyang He (https://chaoyanghe.com). We welcome contributors anywhere in the world. * [2021/02/01] (Award): #NeurIPS 2020# FedML won Best Paper Award at NeurIPS Federated Learning workshop 2020 @@ -28,7 +48,7 @@ FedML is hiring! [Come and join us](https://fedml.ai/careers/)! # **FedML Feature Overview** -![image](/doc/en/_static/image/4animals.png) +![image](./doc/en/_static/image/4animals.png) FedML logo reflects the mission of FedML Inc. FedML aims to build simple and versatile APIs for machine learning running anywhere at any scale. In other words, FedML supports both federated learning for data silos and distributed training for acceleration with MLOps and Open Source support, covering cutting-edge academia research and industrial grade use cases. @@ -210,7 +230,7 @@ Simulation with Message Passing Interface (MPI): Simulation with NCCL-based MPI (the fastest training): - In case your cross-GPU bandwidth is high (e.g., InfiniBand, NVLink, EFA, etc.), we suggest using this NCCL-based MPI FL simulator to accelerate your development. -## **FedML Octopu Exampless** +## **FedML Octopus Examples** Horizontal Federated Learning: - [mqtt_s3_fedavg_mnist_lr_example](./doc/en/cross-silo/examples/mqtt_s3_fedavg_mnist_lr_example.md): an example to illustrate how to run horizontal federated learning in data silos (hospitals, banks, etc.) @@ -226,6 +246,25 @@ Here `hierarchical` means that inside each FL Client (data silo), there are mult - [Federated Learning on Android Smartphones](./doc/en/cross-device/examples/mqtt_s3_fedavg_mnist_lr_example.md) +# FedML on Smartphone and IoTs + + + + + + +
+drawing + +drawing +
+ +See the introduction and tutorial at [FedML/android](./android). + +drawing + +See the introduction and tutorial at [FedML/iot](./iot) + # **MLOps User Guide** [https://open.fedml.ai](https://open.fedml.ai) @@ -236,14 +275,13 @@ FedML MLOps Platform simplifies the workflow of federated learning anywhere at a It enables zero-code, lightweight, cross-platform, and provably secure federated learning. It enables machine learning from decentralized data at various users/silos/edge nodes, without the need to centralize any data to the cloud, hence providing maximum privacy and efficiency. -![image](./doc/en/_static/image/mlops_workflow.png) - +![image](./doc/en/_static/image/MLOps_workflow.png) The above figure shows the workflow. Such a workflow is handled by web UI without the need to handle complex deployment. Check the following live demo for details: ![image](./doc/en/_static/image/mlops_invite.png) -3 Minutes Introduction: [https://www.youtube.com/watch?v=E1k05jd1Tyw](https://www.youtube.com/watch?v=E1k05jd1Tyw) +3 Minutes Introduction: [https://www.youtube.com/watch?v=E1k05jd1Tyw](https://www.youtube.com/watch?v=Xgm0XEaMlVQ) A detailed guidance for the MLOps can be found at [FedML MLOps User Guide](./doc/en/mlops/user_guide.md). @@ -275,7 +313,7 @@ FedML’s core technology is backed by years of cutting-edge research represente 5. AI Applications A Full-stack of Scientific Publications in ML Algorithms, Security/Privacy, Systems, Applications, and Visionary Impacts -Please check [this publication list](./doc/en/resource/papers.md) for details. +Please check [this publication list](./doc/en/resources/papers.md) for details. ## Video (Invited Talks) @@ -298,12 +336,7 @@ Please check [this publication list](./doc/en/resource/papers.md) for details. Our WeChat group exceeds 200 members, please add the following account and ask him to invite you to join. -drawing - -## FAQ - -We organize the frequently asked questions at [https://github.com/FedML-AI/FedML/discussions](https://github.com/FedML-AI/FedML/discussions). -Please feel free to ask questions there. We are happy to discuss on supporting your special demands. +drawing # Contributing diff --git a/android/.gitignore b/android/.gitignore new file mode 100644 index 0000000000..92f7d6fef9 --- /dev/null +++ b/android/.gitignore @@ -0,0 +1,22 @@ +.gradle +/.idea +.cxx + +cpp/build/lightsecagg/build_x86_linux +cpp/build/lightsecagg/build_arm_android_64 +cpp/build/train/build_x86_linux +cpp/build/train/build_arm_android_64 + +*.iml +/local.properties +.DS_Store +/captures +.externalNativeBuild +./idea + +cpp/build_arm_android_64 +cpp/build_x86_linux + +cpp/build/FedMLTrainer +secring.gpg +/build/ diff --git a/android/README.md b/android/README.md new file mode 100644 index 0000000000..20cd3ebd7f --- /dev/null +++ b/android/README.md @@ -0,0 +1,155 @@ +# FedML Android App and SDK + + + + + +
+drawing +
+- Android project root path: https://github.com/FedML-AI/FedML/tree/master/android + +drawing + +The architecture is divided into three vertical layers and multiple horizontal modules: + +### 1. Android APK Layer +- app + +https://github.com/FedML-AI/FedML/tree/master/android/app + + +- fedmlsdk_demo + +https://github.com/FedML-AI/FedML/tree/master/android/fedmlsdk_demo + +### 2. Android SDK layer (Java API + JNI + So library) + +https://github.com/FedML-AI/FedML/tree/master/android/fedmlsdk + + +### 3. MobileNN: FedML Mobile Training Engine Layer (C++, MNN, PyTorch, etc.) + +https://github.com/FedML-AI/FedML/tree/master/android/fedmlsdk/MobileNN + +https://github.com/FedML-AI/MNN + +https://github.com/FedML-AI/pytorch + +## Get Started with FedML Android APP +[https://doc.fedml.ai/cross-device/examples/cross_device_android_example.html](https://doc.fedml.ai/cross-device/examples/cross_device_android_example.html) + +## Get Started with FedML Android SDK + +`android/fedmlsdk_demo` is a short tutorial for integrating Android SDK for your host App. + +1. add repositories by maven + +```groovy + maven { url 'https://s01.oss.sonatype.org/content/repositories/snapshots' } +``` + +2. add dependency in build.gradle + +check `android/fedmlsdk_demo/build.gradle` as an example: + +```groovy + implementation 'ai.fedml:fedml-edge-android:1.0.0-SNAPSHOT' +``` + +3. add FedML account id to meta-data in AndroidManifest.xml + +check `android/fedmlsdk_demo/src/main/AndroidManifest.xml` as an example: + + +```xml + + +``` + +or + +```xml + + +``` + +You can find your account ID at FedML Open Platform (https://open.fedml.ai): +![account](./doc/beehive_account.png) + +4. initial FedML Android SDK on your `Application` class. + +Taking `android/fedmlsdk_demo/src/main/java/ai/fedml/edgedemo/App.java` as an example: +```java +package ai.fedml.edgedemo; + +import android.app.Application; +import android.os.Handler; +import android.os.Looper; + +import ai.fedml.edge.FedEdgeManager; + +public class App extends Application { + private static Handler sHandler = new Handler(Looper.getMainLooper()); + + @Override + public void onCreate() { + super.onCreate(); + + // initial Edge SDK + FedEdgeManager.getFedEdgeApi().init(this); + + // set data path (to prepare data, please check this script `android/data/prepare.sh`) + FedEdgeManager.getFedEdgeApi().setPrivatePath(Environment.getExternalStorageDirectory().getPath() + + "/ai.fedml/device_1/user_0"); + } +} +``` + +## Android SDK APIs +At the current stage, we provide high-level APIs with the following three classes. + + +- ai.fedml.edge.FedEdgeManager + +This is the top APIs in FedML Android SDK, it supports core training engine and related control commands on your Android devices. + +- ai.fedml.edge.OnTrainProgressListener + +This is the message flow to interact between FedML Android SDK and your host APP. + +- ai.fedml.edge.request.RequestManager + +This is used to to connect your Android SDK with FedML Open Platform (https://open.fedml.ai), which helps you to simplify the deployment, edge collaborative training, experimental tracking, and more. + +You can import them in your Java/Android projects as follows. See [android/fedmlsdk_demo/src/main/java/ai/fedml/edgedemo/ui/main/MainFragment.java](fedmlsdk_demo/src/main/java/ai/fedml/edgedemo/ui/main/MainFragment.java) as an example. +``` +import ai.fedml.edge.FedEdgeManager; +import ai.fedml.edge.OnTrainProgressListener; +import ai.fedml.edge.request.RequestManager; +``` + +4. Running Android SDK Demo with MLOps (https://open.fedml.ai) + +Please follow this tutorial (https://doc.fedml.ai/mlops/user_guide.html) to start training using FedML BeeHive Platform. + + + + + + +
+drawing + +drawing +
+ +## How to Run? +https://doc.fedml.ai/cross-device/examples/cross_device_android_example.html + + +## Want More Advanced APIs or Features? +We'd love to listen to your feedback! + +FedML team has rich experience in Android Platform and Federated Learning Algorithmic Research. +If you want advanced feature supports, please send emails to avestimehr@fedml.ai and ch@fedml.ai diff --git a/android/SUBMODULE.md b/android/SUBMODULE.md new file mode 100644 index 0000000000..cb57c81e9f --- /dev/null +++ b/android/SUBMODULE.md @@ -0,0 +1,12 @@ +# git submodule +``` +# add git submodule (please execute under FedML/android folder) +git submodule add https://github.com/FedML-AI/FedMLAndroidSDK.git fedmlsdk + +# git submodule init +git submodule update --init --recursive + +# git submodule update +git submodule update --remote --merge +``` +for more `git submodule` related commands, please refer to https://devconnected.com/how-to-add-and-update-git-submodules/ \ No newline at end of file diff --git a/android/TEST.md b/android/TEST.md new file mode 100644 index 0000000000..ffd55383a8 --- /dev/null +++ b/android/TEST.md @@ -0,0 +1,112 @@ +# Train Flow + +## 1. onStartTrain + +**Received** +topic: flserver_agent/1/start_train + +```json +{ + "groupid": "38", + "clientLearningRate": 0.001, + "partitionMethod": "homo", + "starttime": 1646068794775, + "trainBatchSize": 64, + "edgeids": [ + 17, + 20, + 18, + 21, + 19 + ], + "token": "eyJhbGciOiJIUzI1NiJ9.eyJpZCI6NjAsImFjY291bnQiOiJhbGV4LmxpYW5nIiwibG9naW5UaW1lIjoiMTY0NjA2NTY5MDAwNSIsImV4cCI6MH0.0OTXuMTfxqf2duhkBG1CQDj1UVgconnoSH0PASAEzM4", + "modelName": "resnet56", + "urls": [ + "https://fedmls3.s3.amazonaws.com/025c28be-b464-457a-ab17-851ae60767a9" + ], + "clientOptimizer": "adam", + "userids": [ + "60" + ], + "clientNumPerRound": 3, + "name": "1646068810", + "commRound": 3, + "localEpoch": 1, + "runId": 168, + "id": 169, + "projectid": "56", + "dataset": "cifar10", + "communicationBackend": "MQTT_S3", + "timestamp": "1646068794778" +} +``` + +**Send** +Topic: fedml_168_1 + +```json +{ + "client_status": "ONLINE", + "msg_type": 5, + "receiver": 0, + "sender": 1 +} +``` + +## 2. init Config + +**Received** +Topic: fedml_168_0_1 + +```json +{ + "msg_type": 1, + "sender": 0, + "receiver": 1, + "model_params": "fedml_111_0_39d756ca2-1ce1-44bc-b232-59f0ae054f0e", + "client_idx": "0" +} +``` + +**Send** +Topic: fedml_168_1 + +```json + { + "client_idx": "0", + "model_params": "fedml_111_0_39d756ca2-1ce1-44bc-b232-59f0ae054f0e", + "num_samples": 5, + "msg_type": 3, + "receiver": 0, + "sender": 1 +} +``` + +## 2. Sync Config + +**Received** +Topic: fedml_168_1 + +```json +{ + "msg_type": 2, + "sender": 0, + "receiver": 1, + "model_params": "fedml_111_0_39d756ca2-1ce1-44bc-b232-59f0ae054f0e", + "client_idx": "0" +} +``` + +**Send** +Topic: fedml_168_1 + +```json +{ + "client_idx": "0", + "model_params": "fedml_111_0_39d756ca2-1ce1-44bc-b232-59f0ae054f0e", + "num_samples": 5, + "msg_type": 3, + "receiver": 0, + "sender": 1 +} +``` \ No newline at end of file diff --git a/android/app/.gitignore b/android/app/.gitignore new file mode 100644 index 0000000000..42afabfd2a --- /dev/null +++ b/android/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/app/assets/aria_config.xml b/android/app/assets/aria_config.xml new file mode 100644 index 0000000000..a0f603c1fc --- /dev/null +++ b/android/app/assets/aria_config.xml @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/build.gradle b/android/app/build.gradle new file mode 100644 index 0000000000..9fabeb77ec --- /dev/null +++ b/android/app/build.gradle @@ -0,0 +1,86 @@ +apply plugin: 'com.android.application' + +android { + signingConfigs { + release { + storeFile file('fedml.jks') + storePassword 'fedml0' + keyAlias 'fedml' + keyPassword 'fedml0' + } + } + compileSdkVersion 32 + buildToolsVersion '32.0.0' + ndkVersion '23.1.7779620' + + defaultConfig { + applicationId "ai.fedml" + minSdkVersion 21 + targetSdkVersion 32 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + multiDexEnabled true + signingConfig signingConfigs.release + } + + buildTypes { + debug { + } + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + + compileOptions { + sourceCompatibility 1.8 + targetCompatibility 1.8 + } + + sourceSets { + main { + jniLibs.srcDirs = ['libs'] + } + } + + dataBinding{ + enabled = true + } + +} + +dependencies { + implementation project(':fedmlsdk') + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.3' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' + implementation fileTree(dir: "libs", include: ["*.jar"]) + implementation 'androidx.appcompat:appcompat:1.4.2' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + implementation 'com.google.code.gson:gson:2.9.0' + implementation 'androidx.legacy:legacy-support-v4:1.0.0' + implementation 'com.google.android.material:material:1.6.1' + implementation 'androidx.cardview:cardview:1.0.0' + implementation('com.squareup.okhttp3:okhttp:5.0.0-alpha.7') + + implementation "com.squareup.retrofit2:retrofit:2.9.0" + implementation "com.squareup.retrofit2:converter-gson:2.9.0" + implementation "com.squareup.okio:okio:3.1.0" + implementation 'com.squareup.retrofit2:converter-scalars:2.9.0' + implementation 'com.squareup.okhttp3:logging-interceptor:5.0.0-alpha.7' + + // zxing + implementation 'com.github.yuzhiqiang1993:zxing:2.2.5' + // aria file download + implementation 'me.laoyuyu.aria:core:3.8.16' + annotationProcessor 'me.laoyuyu.aria:compiler:3.8.16' + + annotationProcessor 'org.projectlombok:lombok:1.18.24' + compileOnly 'org.projectlombok:lombok:1.18.24' + + implementation 'com.amazonaws:aws-android-sdk-s3:2.45.0' + implementation 'com.github.bumptech.glide:glide:4.13.2' + annotationProcessor 'com.github.bumptech.glide:compiler:4.13.2' +} \ No newline at end of file diff --git a/android/app/fedml.jks b/android/app/fedml.jks new file mode 100644 index 0000000000..a8ca85315a Binary files /dev/null and b/android/app/fedml.jks differ diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro new file mode 100644 index 0000000000..be1ce06284 --- /dev/null +++ b/android/app/proguard-rules.pro @@ -0,0 +1,109 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile +-ignorewarnings +-keepattributes Exceptions +-keepattributes InnerClasses +-keepattributes SourceFile,LineNumberTable +-keepclasseswithmembernames class * { + native ; +} +-keep public class * extends android.app.Activity +-keep public class * extends android.app.Application +-keep public class * extends android.app.Service +-keep public class * extends android.content.BroadcastReceiver +-keep public class * extends android.content.ContentProvider +-keep public class * extends android.app.backup.BackupAgent +-keep public class * extends android.preference.Preference +-keep public class * extends android.app.Fragment +-keep class androidx.** {*;} +-keep public class * extends androidx.** +-keep interface androidx.** {*;} +-dontwarn androidx.** +-keepclassmembers class * extends android.app.Activity{ + public void *(android.view.View); +} +-keep public class * extends android.view.View{ + *** get*(); + void set*(***); + public (android.content.Context); + public (android.content.Context,android.util.AttributeSet); + public (android.content.Context,android.util.AttributeSet,int); +} +-keepclasseswithmembers class * { + public (android.content.Context, android.util.AttributeSet); + public (android.content.Context, android.util.AttributeSet, int); +} +-dontwarn android.annotation +-keepattributes *Annotation* +# ==================okhttp start=================== +-dontwarn okhttp3.** +-dontwarn okio.** +-dontwarn javax.annotation.** +-dontwarn org.conscrypt.** +# A resource is loaded with a relative path so the package of this class must be preserved. +-keepnames class okhttp3.internal.publicsuffix.PublicSuffixDatabase +# Animal Sniffer compileOnly dependency to ensure APIs are compatible with older versions of Java. +-dontwarn org.codehaus.mojo.animal_sniffer.* +# OkHttp platform used only on JVM and when Conscrypt dependency is available. +-dontwarn okhttp3.internal.platform.ConscryptPlatform +# ==================okhttp end===================== + +# ==================retrofit2 start=================== +# Retain generic type information for use by reflection by converters and adapters. +-keepattributes Signature +# Retain service method parameters. +-keepclassmembernames,allowobfuscation interface * { + @retrofit2.http.* ; +} +# Ignore annotation used for build tooling. +-dontwarn org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement + +# ==================retrofit2 end===================== + +# ==================gson start===================== +-dontwarn com.google.gson.** +-keep class com.google.gson.**{*;} +-keep interface com.google.gson.**{*;} +-dontwarn sun.misc.** +-keepclassmembers,allowobfuscation class * { + @com.google.gson.annotations.SerializedName ; +} +# keep gson entity +-keep class ai.fedml.edge.service.communicator.message.**{*;} +-keep class ai.fedml.edge.request.parameter.**{*;} +-keep class ai.fedml.edge.request.response.**{*;} +# ==================gson end===================== +-keep public class * implements com.bumptech.glide.module.GlideModule +-keep class * extends com.bumptech.glide.module.AppGlideModule { + (...); +} +-keep public enum com.bumptech.glide.load.ImageHeaderParser$** { + **[] $VALUES; + public *; +} +-keep class com.bumptech.glide.load.data.ParcelFileDescriptorRewinder$InternalRewinder { + *** rewind(); +} + +# for DexGuard only +-keepresourcexmlelements manifest/application/meta-data@value=GlideModule +# ==================Glide end===================== \ No newline at end of file diff --git a/android/app/src/androidTest/java/ai/fedml/edge/ExampleInstrumentedTest.java b/android/app/src/androidTest/java/ai/fedml/edge/ExampleInstrumentedTest.java new file mode 100644 index 0000000000..0aad4e9bca --- /dev/null +++ b/android/app/src/androidTest/java/ai/fedml/edge/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package ai.fedml.edge; + +import android.content.Context; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import static org.junit.Assert.assertEquals; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + assertEquals("ai.fedml.ai.fedml.edge.test", appContext.getPackageName()); + } +} \ No newline at end of file diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..4704ef5b7f --- /dev/null +++ b/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/java/ai/fedml/App.java b/android/app/src/main/java/ai/fedml/App.java new file mode 100644 index 0000000000..6cfa94865d --- /dev/null +++ b/android/app/src/main/java/ai/fedml/App.java @@ -0,0 +1,29 @@ +package ai.fedml; + +import android.app.Application; + +import ai.fedml.edge.FedEdgeManager; + + +public class App extends Application { + + + private static App app; + + @Override + public void onCreate() { + super.onCreate(); + app = this; + FedEdgeManager.getFedEdgeApi().init(this); + } + + + public static App getApp() { + return app; + } + + @Override + public void onTerminate() { + super.onTerminate(); + } +} diff --git a/android/app/src/main/java/ai/fedml/CustomGlideModule.java b/android/app/src/main/java/ai/fedml/CustomGlideModule.java new file mode 100644 index 0000000000..ed00a0839c --- /dev/null +++ b/android/app/src/main/java/ai/fedml/CustomGlideModule.java @@ -0,0 +1,9 @@ +package ai.fedml; + +import com.bumptech.glide.annotation.GlideModule; +import com.bumptech.glide.module.AppGlideModule; + +@GlideModule +public final class CustomGlideModule extends AppGlideModule { + +} \ No newline at end of file diff --git a/android/app/src/main/java/ai/fedml/base/AppManager.java b/android/app/src/main/java/ai/fedml/base/AppManager.java new file mode 100644 index 0000000000..1e8a4575e3 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/base/AppManager.java @@ -0,0 +1,132 @@ +package ai.fedml.base; + +import android.app.Activity; +import android.content.Context; + +import java.util.Stack; + +public class AppManager { + + private static Stack activityStack; + private static AppManager instance; + + private AppManager() { + } + + /** + * Single instance, UI does not need to consider multi-threaded synchronization issues + */ + public static AppManager getAppManager() { + if (instance == null) { + instance = new AppManager(); + } + return instance; + } + + /** + * Add Activity to the stack + * + * @param activity + */ + public void addActivity(Activity activity) { + if (activityStack == null) { + activityStack = new Stack(); + } + activityStack.add(activity); + + } + + /** + * Get the current Activity (the Activity at the top of the stack) + */ + public Activity currentActivity() { + if (activityStack == null || activityStack.isEmpty()) { + return null; + } + Activity activity = activityStack.lastElement(); + return activity; + } + + /** + * Get the current Activity (the Activity at the top of the stack) return null when not found + */ + public Activity findActivity(Class cls) { + Activity activity = null; + for (Activity aty : activityStack) { + if (aty.getClass().equals(cls)) { + activity = aty; + break; + } + } + return activity; + } + + /** + * End the current Activity (the Activity at the top of the stack) + */ + public void finishActivity() { + if (activityStack.size() <= 0) { + return; + } + Activity activity = activityStack.lastElement(); + finishActivity(activity); + } + + /** + * End the specified Activity (overload) + */ + public void finishActivity(Activity activity) { + if (activity != null) { + activityStack.remove(activity); + activity.finish(); + activity = null; + } + } + + /** + * Remove the specified Activity and call the finish method elsewhere + */ + public void removeActivity(Activity activity) { + if (activity != null) { + activityStack.remove(activity); + activity = null; + } + } + + /** + * End the specified Activity (overload) + */ + public void finishActivity(Class cls) { + for (Activity activity : activityStack) { + if (activity.getClass().equals(cls)) { + finishActivity(activity); + } + } + } + + /** + * End all the Activities + */ + public void finishAllActivity() { + for (int i = 0, size = activityStack.size(); i < size; i++) { + if (null != activityStack.get(i)) { + activityStack.get(i).finish(); + } + } + activityStack.clear(); + } + + /** + * The application exits, call this method! + */ + public void AppExit(Context context) { + try { + finishAllActivity(); + System.exit(0); + + } catch (Exception e) { + System.exit(0); + } + } + +} diff --git a/android/app/src/main/java/ai/fedml/base/BaseActivity.java b/android/app/src/main/java/ai/fedml/base/BaseActivity.java new file mode 100644 index 0000000000..4ec11ed23b --- /dev/null +++ b/android/app/src/main/java/ai/fedml/base/BaseActivity.java @@ -0,0 +1,35 @@ +package ai.fedml.base; + +import android.app.Activity; +import android.os.Bundle; + +import ai.fedml.utils.StatusBarUtil; +import androidx.annotation.Nullable; + +public class BaseActivity extends Activity { + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + AppManager.getAppManager().addActivity(this); + setStatusBar(); + + } + + protected void setStatusBar() { + //Two things are done here, so that the immersive status bar of the second case mentioned at the beginning can be realized. + // 1.Make the status bar transparent and fill the contentView to the status bar + // 2.Reserve the position of the status bar to prevent the controls on the interface from being too close to the top. + StatusBarUtil.setTransparent(this); + } + + + + + + @Override + protected void onDestroy() { + super.onDestroy(); + AppManager.getAppManager().removeActivity(this); + } +} diff --git a/android/app/src/main/java/ai/fedml/client/RetrofitManager.java b/android/app/src/main/java/ai/fedml/client/RetrofitManager.java new file mode 100644 index 0000000000..b4e0a244ae --- /dev/null +++ b/android/app/src/main/java/ai/fedml/client/RetrofitManager.java @@ -0,0 +1,52 @@ +package ai.fedml.client; + +import java.util.concurrent.TimeUnit; + +import ai.fedml.edge.BuildConfig; +import ai.fedml.edge.request.UserManagerService; +import okhttp3.OkHttpClient; +import okhttp3.logging.HttpLoggingInterceptor; +import retrofit2.Retrofit; +import retrofit2.converter.gson.GsonConverterFactory; + +public final class RetrofitManager { + private static final String BASE_API_SERVER_URL = BuildConfig.MLOPS_SVR; + private static Retrofit retrofit; + private static UserManagerService userManagerService; + private static VersionUpdater versionUpdater; + + private static Retrofit retrofit() { + if (retrofit == null) { + HttpLoggingInterceptor loggingInterceptor = new HttpLoggingInterceptor() + .setLevel(HttpLoggingInterceptor.Level.BASIC); + + OkHttpClient okHttpClient = new OkHttpClient.Builder() + .writeTimeout(30_1000, TimeUnit.MILLISECONDS) + .readTimeout(20_1000, TimeUnit.MILLISECONDS) + .connectTimeout(15_1000, TimeUnit.MILLISECONDS) + .addInterceptor(loggingInterceptor) + .build(); + + retrofit = new Retrofit.Builder() + .baseUrl(BASE_API_SERVER_URL) + .addConverterFactory(GsonConverterFactory.create()) + .client(okHttpClient) + .build(); + } + return retrofit; + } + + public static UserManagerService getOpsUserManager() { + if (userManagerService == null) { + userManagerService = retrofit().create(UserManagerService.class); + } + return userManagerService; + } + + public static VersionUpdater getVersionUpdater() { + if (versionUpdater == null) { + versionUpdater = retrofit().create(VersionUpdater.class); + } + return versionUpdater; + } +} diff --git a/android/app/src/main/java/ai/fedml/client/VersionUpdater.java b/android/app/src/main/java/ai/fedml/client/VersionUpdater.java new file mode 100644 index 0000000000..1aeb733760 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/client/VersionUpdater.java @@ -0,0 +1,10 @@ +package ai.fedml.client; + +import ai.fedml.client.entity.VersionUpdateResponse; +import retrofit2.Call; +import retrofit2.http.GET; + +public interface VersionUpdater { + @GET("/fedmlOpsServer/apk/latestVersion") + Call getLatestVersion(); +} diff --git a/android/app/src/main/java/ai/fedml/client/entity/VersionUpdateResponse.java b/android/app/src/main/java/ai/fedml/client/entity/VersionUpdateResponse.java new file mode 100644 index 0000000000..bc5f898ea0 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/client/entity/VersionUpdateResponse.java @@ -0,0 +1,33 @@ +package ai.fedml.client.entity; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +import ai.fedml.edge.request.response.BaseResponse; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class VersionUpdateResponse extends BaseResponse { + @SerializedName("data") + private List versionList; + + @Data + public static class VersionUpdateInfo { + @SerializedName("createtime") + private String createTime; + @SerializedName("code") + private Integer code; + @SerializedName("name") + private String name; + @SerializedName("id") + private Integer id; + @SerializedName("url") + private String url; + } +} diff --git a/android/app/src/main/java/ai/fedml/ui/GuideActivity.java b/android/app/src/main/java/ai/fedml/ui/GuideActivity.java new file mode 100644 index 0000000000..400c47a04e --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/GuideActivity.java @@ -0,0 +1,88 @@ +package ai.fedml.ui; + +import android.Manifest; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.text.TextUtils; +import android.util.Log; +import android.view.View; + +import java.util.Arrays; + +import androidx.annotation.Nullable; + +import java.io.File; + +import ai.fedml.R; +import ai.fedml.base.AppManager; +import ai.fedml.base.BaseActivity; +import ai.fedml.edge.FedEdgeManager; +import ai.fedml.edge.utils.LogHelper; +import ai.fedml.edge.utils.StorageUtils; + +/** + * Guideline pages + */ +public class GuideActivity extends BaseActivity { + private static final String TAG = "GuideActivity"; + public static final String TRAIN_MODEL_FILE_PATH = StorageUtils.getSdCardPath() + "/ai.fedml/lenet_mnist.mnn"; + public static final String TRAIN_DATA_FILE_PATH = StorageUtils.getSdCardPath() + "/ai.fedml/mnist"; + private final Handler mHandler = new Handler(Looper.getMainLooper()); + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_guide); + initView(); + loadData(); + } + + private void initView() { + Log.d(TAG, "guide ModelPath:" + StorageUtils.getModelPath() + "Dataset:" + StorageUtils.getDatasetPath()); + View guideView = findViewById(R.id.iv_guide); + guideView.setOnClickListener(view -> { + Log.d(TAG, "OnClick guide ModelPath:" + StorageUtils.getModelPath() + + ",Dataset:" + StorageUtils.getDatasetPath()); + Log.d(TAG, "TRAIN_MODEL_FILE_PATH is " + new File(TRAIN_MODEL_FILE_PATH).exists()); + Log.d(TAG, "TRAIN_DATA_FILE_PATH is " + new File(TRAIN_DATA_FILE_PATH).isDirectory()); + }); + } + + private void loadData() { + mHandler.postDelayed(() -> { + final String bindingId = FedEdgeManager.getFedEdgeApi().getBoundEdgeId(); + LogHelper.d("BindingId:%s", bindingId); + if (TextUtils.isEmpty(bindingId)) { + Intent intent = new Intent(); + intent.setClass(GuideActivity.this, ScanCodeActivity.class); + startActivity(intent); + } else { + Intent intent = new Intent(); + intent.setClass(GuideActivity.this, HomeActivity.class); + startActivity(intent); + } + AppManager.getAppManager().finishActivity(); + }, 200); + } + + /** + * Get permission + */ + private void getPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + int REQUEST_CODE_CONTACT = 101; + String[] permissions = {Manifest.permission.WRITE_EXTERNAL_STORAGE, Manifest.permission.READ_EXTERNAL_STORAGE}; + //Verify permission + for (String str : permissions) { + if (this.checkSelfPermission(str) != PackageManager.PERMISSION_GRANTED) { + //Request permission + this.requestPermissions(permissions, REQUEST_CODE_CONTACT); + } + } + } + } +} diff --git a/android/app/src/main/java/ai/fedml/ui/HomeActivity.java b/android/app/src/main/java/ai/fedml/ui/HomeActivity.java new file mode 100644 index 0000000000..d977f296f5 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/HomeActivity.java @@ -0,0 +1,224 @@ +package ai.fedml.ui; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.net.Uri; +import android.os.Build; +import android.os.Bundle; +import android.os.Environment; +import android.provider.Settings; +import android.text.TextUtils; +import android.view.View; +import android.widget.Button; +import android.widget.ImageView; +import android.widget.TextView; + +import ai.fedml.GlideApp; +import ai.fedml.R; +import ai.fedml.base.BaseActivity; +import ai.fedml.edge.FedEdgeManager; +import ai.fedml.edge.OnTrainProgressListener; +import ai.fedml.edge.request.RequestManager; +import ai.fedml.edge.service.communicator.message.MessageDefine; +import ai.fedml.edge.utils.LogHelper; +import ai.fedml.utils.ToastUtils; +import ai.fedml.widget.CompletedProgressView; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +/** + * HomeActivity + */ +public class HomeActivity extends BaseActivity implements View.OnClickListener { + + private static final String TAG = "HomeActivity"; + private Button btn_set_path; + private static final int REQUEST_CODE = 1024; + private TextView mStatusTextView; + private TextView mAccLossTextView; + private CompletedProgressView mProgressView; + private TextView mHyperTextView; + private TextView mNameTextView; + private TextView mEmailTextView; + private TextView mGroupTextView; + private ImageView mAvatarImageView; + private TextView mDeviceAccountInfoTextView; + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_home); + initView(); + loadDate(); + + } + + @Override + protected void onResume() { + super.onResume(); + String path = FedEdgeManager.getFedEdgeApi().getPrivatePath(); + if (!TextUtils.isEmpty(path)) { + btn_set_path.setText(path); + } + } + + private void initView() { + btn_set_path = findViewById(R.id.btn_set_path); + Button btn_unbind = findViewById(R.id.btn_unbind); + + btn_set_path.setOnClickListener(this); + btn_unbind.setOnClickListener(this); + + mStatusTextView = findViewById(R.id.tv_status); + mAccLossTextView = findViewById(R.id.tv_acc_loss); + mProgressView = findViewById(R.id.progress_view); + mHyperTextView = findViewById(R.id.tv_hyper_parameter); + mDeviceAccountInfoTextView = findViewById(R.id.tv_account_info); + mNameTextView = findViewById(R.id.tv_name); + mEmailTextView = findViewById(R.id.tv_email); + mGroupTextView = findViewById(R.id.tv_group); + mAvatarImageView = findViewById(R.id.iv_avatar); + } + + private void loadDate() { + requestPermission(); + getUserInfo(); +// VersionUpdate(); + mDeviceAccountInfoTextView.setText(getString(R.string.account_information, FedEdgeManager.getFedEdgeApi().getBoundEdgeId())); + mProgressView.setProgress(0); + FedEdgeManager.getFedEdgeApi().setEpochLossListener(new OnTrainProgressListener() { + private int mRound = 0; + private int mEpoch = 0; + private float mLoss = 0f; + private float mAccuracy = 0f; + + @Override + public void onEpochLoss(int round, int epoch, float loss) { + mRound = round; + mEpoch = epoch; + mLoss = loss; + runOnUiThread(() -> + mAccLossTextView.setText(getString(R.string.acc_loss_txt, mRound, mEpoch, mAccuracy, mLoss))); + } + + @Override + public void onEpochAccuracy(int round, int epoch, float accuracy) { + mRound = round; + mEpoch = epoch; + mAccuracy = accuracy; + runOnUiThread(() -> + mAccLossTextView.setText(getString(R.string.acc_loss_txt, mRound, mEpoch, mAccuracy, mLoss))); + } + + @Override + public void onProgressChanged(int round, float progress) { + runOnUiThread(() -> + mProgressView.setProgress(Math.round(progress))); + } + }); + FedEdgeManager.getFedEdgeApi().setTrainingStatusListener((status) -> + runOnUiThread(() -> { + if (status == MessageDefine.KEY_CLIENT_STATUS_INITIALIZING) { + mHyperTextView.setText(FedEdgeManager.getFedEdgeApi().getHyperParameters()); + } + mStatusTextView.setText(MessageDefine.CLIENT_STATUS_MAP.get(status)); + })); + } + + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View view) { + switch (view.getId()) { + case R.id.btn_set_path: + Intent intent = new Intent(); + intent.setClass(HomeActivity.this, SetFilePathActivity.class); + startActivity(intent); + break; + case R.id.btn_unbind: + unbound(); + break; + } + } + + private void requestPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + // First determine whether you have permission + if (!Environment.isExternalStorageManager()) { + Intent intent = new Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION); + intent.setData(Uri.parse("package:" + this.getPackageName())); + startActivityForResult(intent, REQUEST_CODE); + } + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // First determine whether you have permission + if (ActivityCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED || + ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.READ_EXTERNAL_STORAGE, Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_CODE); + } + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (requestCode == REQUEST_CODE) { + if (ActivityCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED && + ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED) { + } else { + ToastUtils.show("EXTERNAL STORAGE Permissions failed"); + } + } + } + + @Override + protected void onActivityResult(int requestCode, int resultCode, @Nullable Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == REQUEST_CODE && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + if (Environment.isExternalStorageManager()) { + } else { + ToastUtils.show("Failed to obtain storage permission"); + } + } + } + + private void getUserInfo() { + RequestManager.getUserInfo(userInfo -> { + if (userInfo != null) { + runOnUiThread(() -> { + mNameTextView.setText(String.format("%s %s", userInfo.getLastname(), userInfo.getFirstName())); + mEmailTextView.setText(userInfo.getEmail()); + mGroupTextView.setText(userInfo.getCompany()); + GlideApp.with(HomeActivity.this) + .load(userInfo.getAvatar()) + .circleCrop() + .placeholder(R.mipmap.ic_shijiali) + .into(mAvatarImageView); + }); + } + }); + } + + + private void unbound() { + String bindingId = FedEdgeManager.getFedEdgeApi().getBoundEdgeId(); + LogHelper.d("unbound bindingId:%s", bindingId); + RequestManager.unboundAccount(bindingId, isSuccess -> runOnUiThread(() -> { + if (isSuccess) { + // Jump to scanning page + Intent intent = new Intent(); + intent.setClass(HomeActivity.this, ScanCodeActivity.class); + startActivity(intent); + finish(); + } + })); + } + + @Override + protected void onDestroy() { + super.onDestroy(); + } +} diff --git a/android/app/src/main/java/ai/fedml/ui/ScanCodeActivity.java b/android/app/src/main/java/ai/fedml/ui/ScanCodeActivity.java new file mode 100644 index 0000000000..3138dcfa51 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/ScanCodeActivity.java @@ -0,0 +1 @@ +package ai.fedml.ui; import android.Manifest; import android.content.Intent; import android.content.pm.PackageManager; import android.os.Bundle; import android.os.Handler; import android.os.Looper; import android.text.TextUtils; import android.view.View; import android.widget.Button; import android.widget.EditText; import android.widget.ImageView; import android.widget.LinearLayout; import androidx.annotation.Nullable; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import com.yzq.zxinglibrary.android.CaptureActivity; import com.yzq.zxinglibrary.common.Constant; import ai.fedml.R; import ai.fedml.base.AppManager; import ai.fedml.base.BaseActivity; import ai.fedml.edge.FedEdgeManager; import ai.fedml.edge.request.RequestManager; import ai.fedml.edge.request.parameter.BindingAccountReq; import ai.fedml.edge.utils.DeviceUtils; import ai.fedml.edge.utils.LogHelper; import ai.fedml.utils.ToastUtils; public class ScanCodeActivity extends BaseActivity implements View.OnClickListener { private static final int REQUEST_CODE_SCAN = 0x0000;//Scan ID private EditText edt_account_id; private ImageView img_privacy; private boolean isSelect = false; private final Handler mHandler = new Handler(Looper.getMainLooper()); @Override protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_scan_code); initView(); loadData(); } private void initView() { LinearLayout scanCodeLayout = findViewById(R.id.line_scan_code); edt_account_id = findViewById(R.id.edt_account_id); img_privacy = findViewById(R.id.img_privacy); Button okButton = findViewById(R.id.btn_ok); scanCodeLayout.setOnClickListener(this); img_privacy.setOnClickListener(this); okButton.setOnClickListener(this); } private void loadData() { // meta-data binding Runnable runnable = new Runnable() { @Override public void run() { final String bindingId = FedEdgeManager.getFedEdgeApi().getBoundEdgeId(); if (TextUtils.isEmpty(bindingId)) { mHandler.postDelayed(this, 500); } else { Intent intent = new Intent(); intent.setClass(ScanCodeActivity.this, HomeActivity.class); startActivity(intent); AppManager.getAppManager().finishActivity(); } } }; mHandler.postDelayed(runnable, 500); } @Override public void onClick(View view) { switch (view.getId()) { case R.id.line_scan_code: //Dynamic permission application if (ContextCompat.checkSelfPermission(ScanCodeActivity.this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { ActivityCompat.requestPermissions(ScanCodeActivity.this, new String[]{Manifest.permission.CAMERA}, 1); } else { goScan(); } break; case R.id.img_privacy: if (isSelect) { img_privacy.setBackgroundResource(R.mipmap.ic_privacy_pressed); isSelect = false; } else { img_privacy.setBackgroundResource(R.mipmap.ic_privacy_normal); isSelect = true; } break; case R.id.btn_ok: String id = FedEdgeManager.getFedEdgeApi().getBoundEdgeId(); if (TextUtils.isEmpty(id)) { postBinding(); } else { Intent intent = new Intent(); intent.setClass(ScanCodeActivity.this, HomeActivity.class); startActivity(intent); AppManager.getAppManager().finishActivity(); } break; } } /** * Jump to the scanning code interface to scan the code */ private void goScan() { Intent intent = new Intent(ScanCodeActivity.this, CaptureActivity.class); startActivityForResult(intent, REQUEST_CODE_SCAN); } @Override protected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); // Scan the QR code/barcode and send it back if (requestCode == REQUEST_CODE_SCAN && resultCode == RESULT_OK) { if (data != null) { String content = data.getStringExtra(Constant.CODED_CONTENT); edt_account_id.setText(content); } } } public void postBinding() { String accountId = edt_account_id.getText().toString().trim(); if (TextUtils.isEmpty(accountId)) { ToastUtils.show(R.string.account_input_tip); return; } BindingAccountReq req = BindingAccountReq.builder() .accountId(accountId).deviceId(DeviceUtils.getDeviceId()).build(); RequestManager.bindingAccount(req, data -> runOnUiThread(() -> { LogHelper.i("bindingData.getBindingId() = %s", data); if (data == null) { ToastUtils.show(R.string.retry_tip); return; } FedEdgeManager.getFedEdgeApi().bindEdge(data.getBindingId()); Intent intent = new Intent(); intent.setClass(ScanCodeActivity.this, HomeActivity.class); startActivity(intent); AppManager.getAppManager().finishActivity(); })); } } \ No newline at end of file diff --git a/android/app/src/main/java/ai/fedml/ui/SetFilePathActivity.java b/android/app/src/main/java/ai/fedml/ui/SetFilePathActivity.java new file mode 100644 index 0000000000..e658bca9e5 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/SetFilePathActivity.java @@ -0,0 +1,133 @@ +package ai.fedml.ui; + +import android.annotation.SuppressLint; +import android.os.Bundle; +import android.os.Environment; +import android.view.View; +import android.widget.Button; +import android.widget.TextView; + +import androidx.annotation.Nullable; +import androidx.recyclerview.widget.LinearLayoutManager; +import androidx.recyclerview.widget.RecyclerView; + +import java.io.File; +import java.util.ArrayList; + +import ai.fedml.R; +import ai.fedml.base.AppManager; +import ai.fedml.base.BaseActivity; +import ai.fedml.edge.FedEdgeManager; +import ai.fedml.ui.adapter.FileItem; +import ai.fedml.ui.adapter.RvFilePathAdapter; +import ai.fedml.utils.FileFilter; +import ai.fedml.utils.FileOpenUtils; +import ai.fedml.utils.FormatUtils; + +/** + * SetFilePath + */ +public class SetFilePathActivity extends BaseActivity implements View.OnClickListener, RvFilePathAdapter.OnItemClickListener { + + private TextView tv_path; + private RecyclerView rv_file_path; + private RvFilePathAdapter rvFilePathAdapter; + + private File[] files;// get everything in the directory + private File currentPath; + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_set_file_path); + initView(); + loadData(); + + } + + private void initView() { + Button btn_back = findViewById(R.id.btn_back); + Button btn_save_path = findViewById(R.id.btn_save_path); + tv_path = findViewById(R.id.tv_path); + rv_file_path = findViewById(R.id.rv_file_path); + + btn_back.setOnClickListener(this); + btn_save_path.setOnClickListener(this); + } + + private void loadData() { + ArrayList dataset = new ArrayList<>(); + LinearLayoutManager layoutManager = new LinearLayoutManager(this); + rv_file_path.setLayoutManager(layoutManager); + rvFilePathAdapter = new RvFilePathAdapter(this, dataset); + rv_file_path.setAdapter(rvFilePathAdapter); + rvFilePathAdapter.setOnItemClickListener(this); + + // Get the directory of the sd card + if (Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState())) { + File sd = Environment.getExternalStorageDirectory();// Get the directory of the sd card + // get the contents of the directory + showDir(sd); + } + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View view) { + switch (view.getId()) { + case R.id.btn_save_path: + FedEdgeManager.getFedEdgeApi().setPrivatePath(currentPath.toString()); + AppManager.getAppManager().finishActivity(); + break; + case R.id.btn_back: + // Load parent directory ParentFile: parent directory + File path = currentPath.getParentFile(); + if (path == null || path.toString().equals("/storage/emulated")) { + AppManager.getAppManager().finishActivity(); + } else { + showDir(path); + } + break; + } + } + + + /** + * Load all folders and files and update the interface + */ + @SuppressLint("NotifyDataSetChanged") + private void showDir(File dir) { + // save current location + currentPath = dir; + // Get the contents of the directory (listFiles: get all the contents), and filter the files and folders starting with "." by FileFilter() function + files = dir.listFiles(new FileFilter()); + if (files == null) { + return; + } + rvFilePathAdapter.mSetData.clear(); + for (File file : files) { + FileItem item = FileItem.builder().fileIcon(file.isFile() ? R.mipmap.ic_file : R.mipmap.ic_dir) + .fileName(file.getName()) + .fileSize(FormatUtils.unitConversion(file.length())) + .fileLastModifiedTime(FormatUtils.longToString(file.lastModified())) + .build(); + rvFilePathAdapter.mSetData.add(item); + } + tv_path.setText(currentPath.toString()); + rvFilePathAdapter.notifyDataSetChanged(); + + } + + + @Override + public void onItemClick(View view, int position) { + if (files[position].isFile()) { + // open the file + FileOpenUtils.openFile(this, files[position]); + } else { + // Open the contents of the file directory + // load new data + showDir(files[position]); + } + } +} diff --git a/android/app/src/main/java/ai/fedml/ui/adapter/FileItem.java b/android/app/src/main/java/ai/fedml/ui/adapter/FileItem.java new file mode 100644 index 0000000000..ae00b77b7a --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/adapter/FileItem.java @@ -0,0 +1,13 @@ +package ai.fedml.ui.adapter; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class FileItem { + private int fileIcon; + private String fileName; + private String fileSize; + private String fileLastModifiedTime; +} diff --git a/android/app/src/main/java/ai/fedml/ui/adapter/RvFilePathAdapter.java b/android/app/src/main/java/ai/fedml/ui/adapter/RvFilePathAdapter.java new file mode 100644 index 0000000000..dec577dd0a --- /dev/null +++ b/android/app/src/main/java/ai/fedml/ui/adapter/RvFilePathAdapter.java @@ -0,0 +1,128 @@ +package ai.fedml.ui.adapter; + +import android.content.Context; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.ImageView; +import android.widget.TextView; + +import java.util.List; + +import ai.fedml.R; +import ai.fedml.utils.AppUtils; +import ai.fedml.utils.FormatUtils; +import androidx.annotation.NonNull; +import androidx.recyclerview.widget.RecyclerView; + +/** + * file list adapter + */ +public class RvFilePathAdapter extends RecyclerView.Adapter { + + //Type, use this to determine which layout the recyclerview should use to display + public final int TYPE_EMPTY = 0; + public final int TYPE_NORMAL = 1; + + private final Context mContext; + public List mSetData; + + + public RvFilePathAdapter(Context mContext, List mSetData) { + this.mContext = mContext; + this.mSetData = mSetData; + } + + @Override + public int getItemViewType(int position) { + if (mSetData == null || mSetData.size() <= 0) { + return TYPE_EMPTY; + } + return TYPE_NORMAL; + } + + @NonNull + @Override + public RecyclerView.ViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) { + View view; + //If it is an empty layout type, it will directly return null + if (viewType == TYPE_EMPTY) { + view = LayoutInflater.from(mContext).inflate(R.layout.rv_item_empty, parent, false); + return new EmptyViewHolder(view); + } else { + view = LayoutInflater.from(mContext).inflate(R.layout.rv_item_file, parent, false); + return new BodyViewHolder(view); + } + } + + @Override + public void onBindViewHolder(@NonNull RecyclerView.ViewHolder holder, int position) { + //First determine whether the holder is a custom holder + if (holder instanceof BodyViewHolder) { + holder.itemView.setOnClickListener((View v) -> { + int pos = holder.getLayoutPosition(); + onItemClickListener.onItemClick(holder.itemView, pos); + }); + if (mSetData == null) { + return; + } + FileItem item = mSetData.get(position); + if (item == null) { + return; + } + ((BodyViewHolder) holder).img_icon.setImageResource(item.getFileIcon()); + ((BodyViewHolder) holder).tv_name.setText(item.getFileName()); + ((BodyViewHolder) holder).tv_time.setText(item.getFileLastModifiedTime()); + if (item.getFileIcon() == R.mipmap.ic_dir) { + ((BodyViewHolder) holder).tv_size.setVisibility(View.GONE); + } else { + ((BodyViewHolder) holder).tv_size.setVisibility(View.VISIBLE); + ((BodyViewHolder) holder).tv_size.setText(item.getFileSize()); + } + } + } + + @Override + public int getItemCount() { + if (mSetData == null || mSetData.size() <= 0) { + return 1; + } + return mSetData.size(); + } + + public static class BodyViewHolder extends RecyclerView.ViewHolder { + + ImageView img_icon; + TextView tv_name; + TextView tv_size; + TextView tv_time; + + public BodyViewHolder(@NonNull View itemView) { + super(itemView); + img_icon = itemView.findViewById(R.id.img_icon); + tv_name = itemView.findViewById(R.id.tv_name); + tv_size = itemView.findViewById(R.id.tv_size); + tv_time = itemView.findViewById(R.id.tv_time); + + } + } + + /** + * empty layout + */ + public static class EmptyViewHolder extends RecyclerView.ViewHolder { + public EmptyViewHolder(@NonNull View itemView) { + super(itemView); + } + } + + public interface OnItemClickListener { + void onItemClick(View view, int position); + } + + private OnItemClickListener onItemClickListener; + + public void setOnItemClickListener(OnItemClickListener onItemClickListener) { + this.onItemClickListener = onItemClickListener; + } +} diff --git a/android/app/src/main/java/ai/fedml/utils/AppUtils.java b/android/app/src/main/java/ai/fedml/utils/AppUtils.java new file mode 100644 index 0000000000..72187c3eb4 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/AppUtils.java @@ -0,0 +1,33 @@ +package ai.fedml.utils; + +import android.content.Context; +import android.content.pm.PackageManager; +import android.os.Looper; +import android.view.View; +import android.widget.Toast; + +import java.math.BigDecimal; + + +public class AppUtils { + + + + + + /** + * Get local version number + */ + public static int getVersionCode(Context mContext) { + int versionCode = 0; + try { + //Get the software version number, corresponding to android:versionCode under AndroidManifest.xml + versionCode = mContext.getPackageManager(). + getPackageInfo(mContext.getPackageName(), 0).versionCode; + } catch (PackageManager.NameNotFoundException e) { + e.printStackTrace(); + } + return versionCode; + } + +} diff --git a/android/app/src/main/java/ai/fedml/utils/FileFilter.java b/android/app/src/main/java/ai/fedml/utils/FileFilter.java new file mode 100644 index 0000000000..71ec1d9c15 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/FileFilter.java @@ -0,0 +1,18 @@ +package ai.fedml.utils; + +import java.io.File; + +/** + * FileFilter + */ +public class FileFilter implements java.io.FileFilter { + + @Override + public boolean accept(File pathname) { + // Filter files and folders starting with ".", get the file name, and the prefix is ".", if it is, return false, if not return true + if (pathname.getName().startsWith(".")) { + return false; + } + return true; + } +} diff --git a/android/app/src/main/java/ai/fedml/utils/FileOpenUtils.java b/android/app/src/main/java/ai/fedml/utils/FileOpenUtils.java new file mode 100644 index 0000000000..dae1346aee --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/FileOpenUtils.java @@ -0,0 +1,164 @@ +package ai.fedml.utils; + +import android.content.Context; +import android.content.Intent; +import android.net.Uri; +import android.os.Build; +import android.os.StrictMode; + +import java.io.File; + +public class FileOpenUtils { + private static final String[][] MIME_MapTable={ + //{suffix name, MIME type} + {".3gp", "video/3gpp"}, + {".apk", "application/vnd.android.package-archive"}, + {".asf", "video/x-ms-asf"}, + {".avi", "video/x-msvideo"}, + {".bin", "application/octet-stream"}, + {".bmp", "image/bmp"}, + {".c", "text/plain"}, + {".class", "application/octet-stream"}, + {".conf", "text/plain"}, + {".cpp", "text/plain"}, + {".doc", "application/msword"}, + {".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + {".xls", "application/vnd.ms-excel"}, + {".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, + {".exe", "application/octet-stream"}, + {".gif", "image/gif"}, + {".gtar", "application/x-gtar"}, + {".gz", "application/x-gzip"}, + {".h", "text/plain"}, + {".htm", "text/html"}, + {".html", "text/html"}, + {".jar", "application/java-archive"}, + {".java", "text/plain"}, + {".jpeg", "image/jpeg"}, + {".jpg", "image/jpeg"}, + {".js", "application/x-javascript"}, + {".log", "text/plain"}, + {".m3u", "audio/x-mpegurl"}, + {".m4a", "audio/mp4a-latm"}, + {".m4b", "audio/mp4a-latm"}, + {".m4p", "audio/mp4a-latm"}, + {".m4u", "video/vnd.mpegurl"}, + {".m4v", "video/x-m4v"}, + {".mov", "video/quicktime"}, + {".mp2", "audio/x-mpeg"}, + {".mp3", "audio/x-mpeg"}, + {".mp4", "video/mp4"}, + {".mpc", "application/vnd.mpohun.certificate"}, + {".mpe", "video/mpeg"}, + {".mpeg", "video/mpeg"}, + {".mpg", "video/mpeg"}, + {".mpg4", "video/mp4"}, + {".mpga", "audio/mpeg"}, + {".msg", "application/vnd.ms-outlook"}, + {".ogg", "audio/ogg"}, + {".pdf", "application/pdf"}, + {".png", "image/png"}, + {".pps", "application/vnd.ms-powerpoint"}, + {".ppt", "application/vnd.ms-powerpoint"}, + {".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, + {".prop", "text/plain"}, + {".rc", "text/plain"}, + {".rmvb", "audio/x-pn-realaudio"}, + {".rtf", "application/rtf"}, + {".sh", "text/plain"}, + {".tar", "application/x-tar"}, + {".tgz", "application/x-compressed"}, + {".txt", "text/plain"}, + {".wav", "audio/x-wav"}, + {".wma", "audio/x-ms-wma"}, + {".wmv", "audio/x-ms-wmv"}, + {".wps", "application/vnd.ms-works"}, + {".xml", "text/plain"}, + {".z", "application/x-compress"}, + {".zip", "application/x-zip-compressed"}, + {"", "*/*"} + }; + + /** + * Obtain the corresponding MIME type according to the file suffix. + * @param file + */ + public static String getMIMEType(File file) { + + String type="*/*"; + String fName = file.getName(); + //Get the position of the separator "." before the suffix name in fName. + int dotIndex = fName.lastIndexOf("."); + if(dotIndex < 0){ + return type; + } + /* Get file extension */ + String end=fName.substring(dotIndex,fName.length()).toLowerCase(); + if(end=="")return type; + //Find the corresponding MIME type in the MIME and file type match table. + for(int i=0;i=Build.VERSION_CODES.N) + { + StrictMode.VmPolicy.Builder builder = new StrictMode.VmPolicy.Builder(); + StrictMode.setVmPolicy(builder.build()); + } + +// intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + //Set the Action property of the intent + intent.setAction(Intent.ACTION_VIEW); + //Get the MIME type of the file file + String type = getMIMEType(file); + //Set the data and Type properties of the intent. + intent.setDataAndType(/*uri*/Uri.fromFile(file), type); + //Jump + context.startActivity(intent); + + } + + + public static boolean isImage(File file) + { + String type = getType(file); + if (".jpg".equals(type)||".png".equals(type)) + { + return true; + } + return false; + + } + +} diff --git a/android/app/src/main/java/ai/fedml/utils/FormatUtils.java b/android/app/src/main/java/ai/fedml/utils/FormatUtils.java new file mode 100644 index 0000000000..1e7d6bf724 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/FormatUtils.java @@ -0,0 +1,61 @@ +package ai.fedml.utils; + +import android.content.Context; +import android.text.format.DateFormat; + +import java.math.BigDecimal; +import java.util.Date; + +import ai.fedml.edge.service.ContextHolder; + +public class FormatUtils { + private FormatUtils() { + } + + /** + * Convert long type to String type + * + * @param milSecond currentTime time of type long to convert + * @return LongDateFormat + */ + public static String longToString(long milSecond) { + Context context = ContextHolder.getAppContext(); + return DateFormat.getLongDateFormat(context).format(new Date(milSecond)); + } + + /** + * byte convert to kb、mb、gb、tb + * + * @param average size in bytes + * @return File size + */ + public static String unitConversion(long average) { + double temp = average; + if (temp < 1024) { + BigDecimal result1 = new BigDecimal(temp); + return result1.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "B"; + } + temp = temp / 1024; + if (temp < 1024) { + BigDecimal result1 = new BigDecimal(temp); + return result1.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "KB"; + } + + temp = temp / 1024; + if (temp < 1024) { + BigDecimal result1 = new BigDecimal(temp); + return result1.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "MB"; + } + temp = temp / 1024; + if (temp < 1024) { + BigDecimal result1 = new BigDecimal(temp); + return result1.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "GB"; + } + temp = temp / 1024; + if (temp < 1024) { + BigDecimal result1 = new BigDecimal(temp); + return result1.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "TB"; + } + return "0"; + } +} diff --git a/android/app/src/main/java/ai/fedml/utils/StatusBarUtil.java b/android/app/src/main/java/ai/fedml/utils/StatusBarUtil.java new file mode 100644 index 0000000000..610b504936 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/StatusBarUtil.java @@ -0,0 +1,69 @@ +package ai.fedml.utils; + +import android.annotation.TargetApi; +import android.app.Activity; +import android.graphics.Color; +import android.os.Build; +import android.view.View; +import android.view.ViewGroup; +import android.view.WindowManager; + +/** + * Adapt to full screen status bar + */ +public class StatusBarUtil { + + + /** + * Make the status bar fully transparent + * + * @param activity needs to be set up + */ + public static void setTransparent(Activity activity) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { + return; + } + transparentStatusBar(activity); + setRootView(activity); + } + + /** + * make the status bar transparent + */ + @TargetApi(Build.VERSION_CODES.KITKAT) + private static void transparentStatusBar(Activity activity) { + // Set Android 6.0 + to achieve status bar text color and icon light black + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + activity.getWindow().getDecorView().setSystemUiVisibility( + View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN | View.SYSTEM_UI_FLAG_LIGHT_STATUS_BAR); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + activity.getWindow().addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS); + activity.getWindow().clearFlags(WindowManager.LayoutParams.FLAG_TRANSLUCENT_STATUS); + //You need to set this flag contentView to extend to the status bar, and the bottom will also extend past +// activity.getWindow().addFlags(WindowManager.LayoutParams.FLAG_TRANSLUCENT_NAVIGATION); + //The status bar is overlaid on the contentView, and the transparency is set to make the background of the contentView show through + activity.getWindow().setStatusBarColor(Color.TRANSPARENT); + } else { + //Let the contentView extend to the status bar and set the status bar color to be transparent + activity.getWindow().addFlags(WindowManager.LayoutParams.FLAG_TRANSLUCENT_STATUS); + } + } + + /** + * Set root layout parameters + */ + private static void setRootView(Activity activity) { + ViewGroup parent = (ViewGroup) activity.findViewById(android.R.id.content); + for (int i = 0, count = parent.getChildCount(); i < count; i++) { + View childView = parent.getChildAt(i); + if (childView instanceof ViewGroup) { + childView.setFitsSystemWindows(true); + ((ViewGroup) childView).setClipToPadding(true); + } + } + } + + + +} diff --git a/android/app/src/main/java/ai/fedml/utils/ToastUtils.java b/android/app/src/main/java/ai/fedml/utils/ToastUtils.java new file mode 100644 index 0000000000..50b9a2a8a7 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/utils/ToastUtils.java @@ -0,0 +1,52 @@ +package ai.fedml.utils; + +import android.content.Context; +import android.content.res.Resources; +import android.os.Handler; +import android.os.Looper; +import android.widget.Toast; + +import ai.fedml.edge.service.ContextHolder; + +public class ToastUtils { + private static final Handler sMainHandler = new Handler(Looper.getMainLooper()); + + /** + * display Toast + * + * @param text + */ + public static void show(CharSequence text) { + Runnable toastRunnable = () -> { + Context context = ContextHolder.getAppContext(); + if (text == null || text.equals("")) return; + // If the displayed text exceeds 10, display the long toast, otherwise display the short toast + int duration = Toast.LENGTH_SHORT; + if (text.length() > 20) { + duration = Toast.LENGTH_LONG; + } + Toast.makeText(context, text, duration).show(); + }; + if (Looper.getMainLooper() == Looper.myLooper()) { + toastRunnable.run(); + } else { + sMainHandler.post(toastRunnable); + } + + } + + /** + * Display Toast + * + * @param id If the correct string id is passed in, the corresponding string will be displayed, if not, an integer string will be displayed + */ + public static void show(int id) { + try { + // if this is a resource id + show(ContextHolder.getAppContext().getResources().getText(id)); + } catch (Resources.NotFoundException ignored) { + // if this is an int type + show(String.valueOf(id)); + } + } +} diff --git a/android/app/src/main/java/ai/fedml/widget/CircleImageView.java b/android/app/src/main/java/ai/fedml/widget/CircleImageView.java new file mode 100644 index 0000000000..98e4394f83 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/widget/CircleImageView.java @@ -0,0 +1,84 @@ +package ai.fedml.widget; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.BitmapShader; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.Shader; +import android.graphics.drawable.BitmapDrawable; +import android.graphics.drawable.Drawable; +import android.util.AttributeSet; + +/** + * custom circular image + */ +public class CircleImageView extends androidx.appcompat.widget.AppCompatImageView { + + private Paint mPaint = new Paint(); //brush + + private int mRadius; //The radius of the circular image + + private float mScale; //The zoom ratio of the image + + public CircleImageView(Context context) { + super(context); + } + + public CircleImageView(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public CircleImageView(Context context, AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + } + + @Override + protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + //Because it is a circular image, the width and height should be consistent + int size = Math.min(getMeasuredWidth(), getMeasuredHeight()); + mRadius = size / 2; + + setMeasuredDimension(size, size); + } + + @Override + protected void onDraw(Canvas canvas) { + + Bitmap bitmap = drawableToBitmap(getDrawable()); + + //Initialize BitmapShader, pass in the bitmap object + BitmapShader bitmapShader = new BitmapShader(bitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP); + + //Calculate scaling + mScale = (mRadius * 2.0f) / Math.min(bitmap.getHeight(), bitmap.getWidth()); + + Matrix matrix = new Matrix(); + matrix.setScale(mScale, mScale); + bitmapShader.setLocalMatrix(matrix); + + + mPaint.setShader(bitmapShader); + + //Draw a circle, specify the center point coordinates, radius, brush + canvas.drawCircle(mRadius, mRadius, mRadius, mPaint); + } + + //Write a drawble to BitMap method + private Bitmap drawableToBitmap(Drawable drawable) { + if (drawable instanceof BitmapDrawable) { + BitmapDrawable bd = (BitmapDrawable) drawable; + return bd.getBitmap(); + } + int w = drawable.getIntrinsicWidth(); + int h = drawable.getIntrinsicHeight(); + Bitmap bitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + Canvas canvas = new Canvas(bitmap); + drawable.setBounds(0, 0, w, h); + drawable.draw(canvas); + return bitmap; + } + +} diff --git a/android/app/src/main/java/ai/fedml/widget/CompletedProgressView.java b/android/app/src/main/java/ai/fedml/widget/CompletedProgressView.java new file mode 100644 index 0000000000..461eff9c95 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/widget/CompletedProgressView.java @@ -0,0 +1,165 @@ +package ai.fedml.widget; + +import android.content.Context; +import android.content.res.TypedArray; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.RectF; +import android.text.TextUtils; +import android.util.AttributeSet; +import android.view.View; + +import ai.fedml.R; + +/** + * Custom circular progress bar + */ +public class CompletedProgressView extends View { + + // total progress + private static final int TOTAL_PROGRESS = 100; + // Paintbrush for drawing a filled circle + private Paint mCirclePaint; + // brush for drawing circles + private Paint mRingPaint; + // The background color of the brush for drawing the ring + private Paint mRingPaintBg; + // brush for drawing fonts + private Paint mTextPaint; + // circle color + private int mCircleColor; + // ring color + private int mRingColor; + // Ring background color + private int mRingBgColor; + // radius + private float mRadius; + // Ring radius + private float mRingRadius; + // Ring width + private float mStrokeWidth; + // word height + private float mTxtHeight; + // current progress + private int mProgress; + private RectF mOuterRect; + private String mStatus; + + public CompletedProgressView(Context context, AttributeSet attrs) { + super(context, attrs); + // Get custom properties + initAttrs(context, attrs); + initVariable(); + } + + //properties + private void initAttrs(Context context, AttributeSet attrs) { + TypedArray typeArray = context.getTheme().obtainStyledAttributes(attrs, + R.styleable.TasksCompletedView, 0, 0); + mRadius = typeArray.getDimension(R.styleable.TasksCompletedView_radius, 80); + mStrokeWidth = typeArray.getDimension(R.styleable.TasksCompletedView_strokeWidth, 10); + mCircleColor = typeArray.getColor(R.styleable.TasksCompletedView_circleColor, 0xFFFFFFFF); + mRingColor = typeArray.getColor(R.styleable.TasksCompletedView_ringColor, 0xFFFFFFFF); + mRingBgColor = typeArray.getColor(R.styleable.TasksCompletedView_ringBgColor, 0xFFFFFFFF); + mProgress = typeArray.getInteger(R.styleable.TasksCompletedView_progress, 0); + + mRingRadius = mRadius + mStrokeWidth / 2; + } + + //Initialize brush + private void initVariable() { + //inner circle + mCirclePaint = new Paint(); + mCirclePaint.setAntiAlias(true); + mCirclePaint.setColor(mCircleColor); + mCirclePaint.setStyle(Paint.Style.FILL); + + //Outer arc background + mRingPaintBg = new Paint(); + mRingPaintBg.setAntiAlias(true); + mRingPaintBg.setColor(mRingBgColor); + mRingPaintBg.setStyle(Paint.Style.STROKE); + mRingPaintBg.setStrokeWidth(mStrokeWidth); + + + //Outer arc + mRingPaint = new Paint(); + mRingPaint.setAntiAlias(true); + mRingPaint.setColor(mRingColor); + mRingPaint.setStyle(Paint.Style.STROKE); + mRingPaint.setStrokeWidth(mStrokeWidth); + //mRingPaint.setStrokeCap(Paint.Cap.ROUND);//Set the line style, there are circles and squares + + //middle word + mTextPaint = new Paint(); + mTextPaint.setAntiAlias(true); + mTextPaint.setStyle(Paint.Style.FILL); + mTextPaint.setColor(getResources().getColor(R.color.color_3C4043)); + mTextPaint.setTextSize(mRadius / 2); + + Paint.FontMetrics fm = mTextPaint.getFontMetrics(); + mTxtHeight = (int) Math.ceil(fm.descent - fm.ascent); + } + + @Override + protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + } + + //draw + @Override + protected void onDraw(Canvas canvas) { + // The x-coordinate of the center of the circle + int mXCenter = getWidth() / 2; + // The y coordinate of the center of the circle + int mYCenter = getHeight() / 2; + + //inner circle + canvas.drawCircle(mXCenter, mYCenter, mRadius, mCirclePaint); + + //Outer arc background + if (mOuterRect == null) { + mOuterRect = new RectF(); + mOuterRect.left = (mXCenter - mRingRadius); + mOuterRect.top = (mYCenter - mRingRadius); + mOuterRect.right = mRingRadius * 2 + (mXCenter - mRingRadius); + mOuterRect.bottom = mRingRadius * 2 + (mYCenter - mRingRadius); + } + canvas.drawArc(mOuterRect, 0, 360, false, mRingPaintBg); + + //The ellipse object where the arc is located, the starting angle of the arc, the angle of the arc, whether to display the radius connection + + //Outer arc + if (mProgress > 0) { + canvas.drawArc(mOuterRect, -90, ((float) mProgress / TOTAL_PROGRESS) * 360, false, mRingPaint); // + } + + //fonts + String txt = mStatus; + if (TextUtils.isEmpty(mStatus) && mProgress > 0) { + txt = mProgress + "%"; + } + if (!TextUtils.isEmpty(txt)) { + // word length + float mTxtWidth = mTextPaint.measureText(txt, 0, txt.length()); + canvas.drawText(txt, mXCenter - mTxtWidth / 2, mYCenter + mTxtHeight / 4, mTextPaint); + } + } + + //set the progress + public void setProgress(int progress) { + mProgress = progress; + mStatus = null; + postInvalidate(); + } + + /** + * Set text status + * + * @param status status + */ + public void setStatus(String status) { + mStatus = status; + postInvalidate(); + } +} diff --git a/android/app/src/main/java/ai/fedml/widget/CouponTextView.java b/android/app/src/main/java/ai/fedml/widget/CouponTextView.java new file mode 100644 index 0000000000..ef9cc4df5d --- /dev/null +++ b/android/app/src/main/java/ai/fedml/widget/CouponTextView.java @@ -0,0 +1,65 @@ +package ai.fedml.widget; + +import android.content.Context; +import android.content.res.TypedArray; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.RectF; +import android.util.AttributeSet; + +import ai.fedml.R; +import androidx.core.content.ContextCompat; + +/** + * custom bump layout + */ +public class CouponTextView extends androidx.appcompat.widget.AppCompatTextView { + + private Paint mPaint; + private Context mContext; + private int mColor; + private int mHeight; + private RectF mRectF; + + public CouponTextView(Context context) { + this(context, null); + + } + + public CouponTextView(Context context, AttributeSet attrs) { + this(context, attrs, 0); + + } + + public CouponTextView(Context context, AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + TypedArray array = context.obtainStyledAttributes(attrs, R.styleable.CouponTextView); + mColor = ContextCompat.getColor(context, R.color.white); + mColor = array.getColor(R.styleable.CouponTextView_bg_color, mColor); + mHeight = array.getDimensionPixelSize(R.styleable.CouponTextView_android_height, 30); + mContext = context; + initPaint(); + array.recycle(); + } + + private void initPaint() { + mPaint =new Paint(); + mPaint.setColor(mColor); + mPaint.setStrokeWidth(12f); + mPaint.setAntiAlias(true); + } + + @Override + protected void onDraw(Canvas canvas) { + + super.onDraw(canvas); + if(mRectF == null) { + mRectF = new RectF(0, 0, getMeasuredWidth(), mHeight); + } + canvas.drawRect(mRectF, mPaint); + mPaint.setColor(ContextCompat.getColor(mContext, R.color.color_F5F6FA)); + canvas.drawCircle(getMeasuredWidth()/2, 0,50, mPaint); + } + + +} diff --git a/android/app/src/main/java/ai/fedml/widget/PopupwindNormal.java b/android/app/src/main/java/ai/fedml/widget/PopupwindNormal.java new file mode 100644 index 0000000000..4c928df577 --- /dev/null +++ b/android/app/src/main/java/ai/fedml/widget/PopupwindNormal.java @@ -0,0 +1,74 @@ +package ai.fedml.widget; + +import android.content.Context; +import android.graphics.drawable.ColorDrawable; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.Button; +import android.widget.PopupWindow; +import android.widget.TextView; + +import ai.fedml.R; + + +public class PopupwindNormal extends PopupWindow { + + private View mView; + private TextView tv_title; + private TextView tv_content; + + private Button btn_cancel; + private Button btn_ok; + + + /** + * + * @param context + * @param onClickListener + * @param title + * @param content + */ + public PopupwindNormal(Context context, View.OnClickListener onClickListener, String title, String content) { + super(context); + + LayoutInflater inflater = (LayoutInflater) context.getSystemService(Context.LAYOUT_INFLATER_SERVICE); + mView = inflater.inflate(R.layout.popup_normal, null); + + tv_title = mView.findViewById(R.id.tv_title); + tv_content = mView.findViewById(R.id.tv_content); + btn_cancel = mView.findViewById(R.id.btn_cancel); + btn_ok = mView.findViewById(R.id.btn_ok); + + btn_ok.setOnClickListener(onClickListener); + btn_cancel.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + dismiss(); + } + }); + + tv_title.setText(title); + tv_content.setText(content); + + + + + // Set the View of PopupWindow + this.setContentView(mView); + // Set the width of the PopupWindow pop-up form + this.setWidth(ViewGroup.LayoutParams.MATCH_PARENT); + // Set the height of the PopupWindow pop-up form + this.setHeight(ViewGroup.LayoutParams.MATCH_PARENT); + // Set the PopupWindow pop-up form to be clickable + this.setFocusable(true); + // Set PopupWindow pop-up form animation effect + this.setAnimationStyle(R.style.AnimFadePopup); + // Instantiate a ColorDrawable with a color of black with 25% opacity + ColorDrawable dw = new ColorDrawable(0x40000000); + // Set the background of the PopupWindow popup form + this.setBackgroundDrawable(dw); + + } + +} diff --git a/android/app/src/main/res/anim/fade_in.xml b/android/app/src/main/res/anim/fade_in.xml new file mode 100644 index 0000000000..d4193790cf --- /dev/null +++ b/android/app/src/main/res/anim/fade_in.xml @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/android/app/src/main/res/anim/fade_out.xml b/android/app/src/main/res/anim/fade_out.xml new file mode 100644 index 0000000000..6b1b69a51a --- /dev/null +++ b/android/app/src/main/res/anim/fade_out.xml @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/android/app/src/main/res/drawable/border_edit.xml b/android/app/src/main/res/drawable/border_edit.xml new file mode 100644 index 0000000000..c5569dde6e --- /dev/null +++ b/android/app/src/main/res/drawable/border_edit.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/res/drawable/radius_whit_bg.xml b/android/app/src/main/res/drawable/radius_whit_bg.xml new file mode 100644 index 0000000000..034ba9173f --- /dev/null +++ b/android/app/src/main/res/drawable/radius_whit_bg.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/res/layout/activity_guide.xml b/android/app/src/main/res/layout/activity_guide.xml new file mode 100644 index 0000000000..699bc1c304 --- /dev/null +++ b/android/app/src/main/res/layout/activity_guide.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/res/layout/activity_home.xml b/android/app/src/main/res/layout/activity_home.xml new file mode 100644 index 0000000000..1698e18b09 --- /dev/null +++ b/android/app/src/main/res/layout/activity_home.xml @@ -0,0 +1,206 @@ + + + + + + + + +