Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update gRPC and ProtoBuf dependencies #2814

Merged
merged 31 commits into from
Jan 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
4d4a68c
Add Python 3.11 to framework tests
danieljanes Sep 19, 2023
05aae22
Merge branch 'main' into ci-py311
danieljanes Oct 7, 2023
cbf344c
Merge branch 'main' into ci-py311
danieljanes Oct 8, 2023
2cd3e54
Merge branch 'main' into ci-py311
danieljanes Nov 5, 2023
acbde35
Merge branch 'main' into ci-py311
danieljanes Nov 6, 2023
b64ce84
Upgrade grpcio
charlesbvll Nov 6, 2023
3dfd048
Merge branch 'main' into ci-py311
charlesbvll Nov 6, 2023
e413d3c
Merge branch 'main' into ci-py311
charlesbvll Nov 6, 2023
1550043
Update protobuf
charlesbvll Nov 6, 2023
5fb7f9e
Recompile protofiles
charlesbvll Nov 6, 2023
ec1544b
Merge branch 'main' into ci-py311
charlesbvll Nov 6, 2023
2dbc188
Add pylint-protobuf library
charlesbvll Nov 6, 2023
de89d18
Try to update pylint
charlesbvll Nov 7, 2023
f2ddfe0
Merge branch 'main' into ci-py311
charlesbvll Nov 7, 2023
2b49292
Merge branch 'main' into ci-py311
charlesbvll Nov 7, 2023
e13c125
Merge branch 'main' into ci-py311
charlesbvll Jan 17, 2024
88a77fc
Add None timeout
charlesbvll Jan 17, 2024
a3b66e4
Recompile protos
charlesbvll Jan 17, 2024
4a40069
Recompile protos with correct version
charlesbvll Jan 17, 2024
471d01e
Merge branch 'main' into ci-py311
charlesbvll Jan 17, 2024
51a0933
Revert timeout changes
charlesbvll Jan 17, 2024
981d27a
Merge branch 'main' into ci-py311
charlesbvll Jan 18, 2024
41d0004
Add pylint disable statements
charlesbvll Jan 18, 2024
89f5ac2
Sort imports
charlesbvll Jan 18, 2024
5b864bc
Fix no-member errors
charlesbvll Jan 18, 2024
3557a2f
Place disables correctly
charlesbvll Jan 18, 2024
a02beed
Update protobuf and grpcio
charlesbvll Jan 18, 2024
5298d35
Recompile protos
charlesbvll Jan 18, 2024
4ff960f
Remove py311 from PR
charlesbvll Jan 18, 2024
e727228
Merge branch 'main' into ignore-pylint-proto
danieljanes Jan 18, 2024
2d91e30
Merge branch 'main' into ignore-pylint-proto
danieljanes Jan 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ flower-client = "flwr.client:run_client"
python = "^3.8"
# Mandatory dependencies
numpy = "^1.21.0"
grpcio = "^1.48.2,!=1.52.0"
protobuf = "^3.19.0"
grpcio = "^1.60.0"
protobuf = "^4.25.2"
cryptography = "^41.0.2"
pycryptodome = "^3.18.0"
iterators = "^0.0.2"
Expand Down Expand Up @@ -93,7 +93,7 @@ flake8 = "==5.0.4"
pytest = "==7.4.3"
pytest-cov = "==4.1.0"
pytest-watch = "==4.2.0"
grpcio-tools = "==1.48.2"
grpcio-tools = "==1.60.0"
mypy-protobuf = "==3.2.0"
jupyterlab = "==4.0.9"
rope = "==1.11.0"
Expand Down
2 changes: 1 addition & 1 deletion src/py/flwr/client/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
TRANSPORT_TYPES,
)
from flwr.common.logger import log, warn_experimental_feature
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611

from .flower import load_flower_callable
from .grpc_client.connection import grpc_connection
Expand Down
11 changes: 7 additions & 4 deletions src/py/flwr/client/grpc_client/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,13 @@
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
from flwr.common.grpc import create_channel
from flwr.common.logger import log
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
from flwr.proto.transport_pb2_grpc import FlowerServiceStub
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes # pylint: disable=E0611
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
ServerMessage,
)
from flwr.proto.transport_pb2_grpc import FlowerServiceStub # pylint: disable=E0611

# The following flags can be uncommented for debugging. Other possible values:
# https://github.com/grpc/grpc/blob/master/doc/environment_variables.md
Expand Down
7 changes: 5 additions & 2 deletions src/py/flwr/client/grpc_client/connection_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@

import grpc

from flwr.proto.task_pb2 import Task, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
from flwr.proto.task_pb2 import Task, TaskRes # pylint: disable=E0611
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
ServerMessage,
)
from flwr.server.client_manager import SimpleClientManager
from flwr.server.fleet.grpc_bidi.grpc_server import start_grpc_server

Expand Down
8 changes: 4 additions & 4 deletions src/py/flwr/client/grpc_rere_client/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
from flwr.common.grpc import create_channel
from flwr.common.logger import log, warn_experimental_feature
from flwr.proto.fleet_pb2 import (
from flwr.proto.fleet_pb2 import ( # pylint: disable=E0611
CreateNodeRequest,
DeleteNodeRequest,
PullTaskInsRequest,
PushTaskResRequest,
)
from flwr.proto.fleet_pb2_grpc import FleetStub
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.fleet_pb2_grpc import FleetStub # pylint: disable=E0611
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611

KEY_NODE = "node"
KEY_TASK_INS = "current_task_ins"
Expand Down
13 changes: 11 additions & 2 deletions src/py/flwr/client/message_handler/message_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,17 @@
from flwr.client.secure_aggregation import SecureAggregationHandler
from flwr.client.typing import ClientFn
from flwr.common import serde
from flwr.proto.task_pb2 import SecureAggregation, Task, TaskIns, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, Reason, ServerMessage
from flwr.proto.task_pb2 import ( # pylint: disable=E0611
SecureAggregation,
Task,
TaskIns,
TaskRes,
)
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
Reason,
ServerMessage,
)


class UnexpectedServerMessage(Exception):
Expand Down
11 changes: 8 additions & 3 deletions src/py/flwr/client/message_handler/message_handler_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,14 @@
serde,
typing,
)
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, Code, ServerMessage, Status
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes # pylint: disable=E0611
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
Code,
ServerMessage,
Status,
)

from .message_handler import handle, handle_control_message

Expand Down
11 changes: 7 additions & 4 deletions src/py/flwr/client/message_handler/task_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@

from typing import Optional

from flwr.proto.fleet_pb2 import PullTaskInsResponse
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
from flwr.proto.fleet_pb2 import PullTaskInsResponse # pylint: disable=E0611
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes # pylint: disable=E0611
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
ServerMessage,
)


def validate_task_ins(task_ins: TaskIns, discard_reconnect_ins: bool) -> bool:
Expand Down
14 changes: 11 additions & 3 deletions src/py/flwr/client/message_handler/task_handler_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,17 @@
validate_task_res,
wrap_client_message_in_task_res,
)
from flwr.proto.fleet_pb2 import PullTaskInsResponse
from flwr.proto.task_pb2 import SecureAggregation, Task, TaskIns, TaskRes
from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
from flwr.proto.fleet_pb2 import PullTaskInsResponse # pylint: disable=E0611
from flwr.proto.task_pb2 import ( # pylint: disable=E0611
SecureAggregation,
Task,
TaskIns,
TaskRes,
)
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
ServerMessage,
)


def test_validate_task_ins_no_task() -> None:
Expand Down
2 changes: 1 addition & 1 deletion src/py/flwr/client/middleware/utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from flwr.client.run_state import RunState
from flwr.client.typing import Bwd, FlowerCallable, Fwd, Layer
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611

from .utils import make_ffn

Expand Down
2 changes: 1 addition & 1 deletion src/py/flwr/client/node_state_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from flwr.client.node_state import NodeState
from flwr.client.run_state import RunState
from flwr.proto.task_pb2 import TaskIns
from flwr.proto.task_pb2 import TaskIns # pylint: disable=E0611


def _run_dummy_task(state: RunState) -> RunState:
Expand Down
6 changes: 3 additions & 3 deletions src/py/flwr/client/rest_client/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from flwr.common import GRPC_MAX_MESSAGE_LENGTH
from flwr.common.constant import MISSING_EXTRA_REST
from flwr.common.logger import log
from flwr.proto.fleet_pb2 import (
from flwr.proto.fleet_pb2 import ( # pylint: disable=E0611
CreateNodeRequest,
CreateNodeResponse,
DeleteNodeRequest,
Expand All @@ -38,8 +38,8 @@
PushTaskResRequest,
PushTaskResResponse,
)
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611

try:
import requests
Expand Down
2 changes: 1 addition & 1 deletion src/py/flwr/client/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from typing import Callable

from flwr.client.run_state import RunState
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611

from .client import Client as Client

Expand Down
4 changes: 2 additions & 2 deletions src/py/flwr/common/serde.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

from typing import Any, Dict, List, MutableMapping, cast

from flwr.proto.task_pb2 import Value
from flwr.proto.transport_pb2 import (
from flwr.proto.task_pb2 import Value # pylint: disable=E0611
from flwr.proto.transport_pb2 import ( # pylint: disable=E0611
ClientMessage,
Code,
Parameters,
Expand Down
10 changes: 5 additions & 5 deletions src/py/flwr/common/serde_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from typing import Dict, Union, cast

from flwr.common import typing
from flwr.proto import transport_pb2 as pb2
from flwr.proto import transport_pb2 as pb2 # pylint: disable=E0611

from .serde import (
named_values_from_proto,
Expand Down Expand Up @@ -50,8 +50,8 @@ def test_serialisation_deserialisation() -> None:
def test_status_to_proto() -> None:
"""Test status message (de-)serialization."""
# Prepare
code_msg = pb2.Code.OK
status_msg = pb2.Status(code=code_msg, message="Success")
code_msg = pb2.Code.OK # pylint: disable=E1101
status_msg = pb2.Status(code=code_msg, message="Success") # pylint: disable=E1101

code = typing.Code.OK
status = typing.Status(code=code, message="Success")
Expand All @@ -66,8 +66,8 @@ def test_status_to_proto() -> None:
def test_status_from_proto() -> None:
"""Test status message (de-)serialization."""
# Prepare
code_msg = pb2.Code.OK
status_msg = pb2.Status(code=code_msg, message="Success")
code_msg = pb2.Code.OK # pylint: disable=E1101
status_msg = pb2.Status(code=code_msg, message="Success") # pylint: disable=E1101

code = typing.Code.OK
status = typing.Status(code=code, message="Success")
Expand Down
8 changes: 5 additions & 3 deletions src/py/flwr/driver/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from flwr.common import EventType, event
from flwr.common.address import parse_address
from flwr.common.logger import log
from flwr.proto import driver_pb2
from flwr.proto import driver_pb2 # pylint: disable=E0611
from flwr.server.app import ServerConfig, init_defaults, run_fl
from flwr.server.client_manager import ClientManager
from flwr.server.history import History
Expand Down Expand Up @@ -171,7 +171,9 @@ def update_client_manager(
`client_manager.unregister()`.
"""
# Request for run_id
run_id = driver.create_run(driver_pb2.CreateRunRequest()).run_id
run_id = driver.create_run(
driver_pb2.CreateRunRequest() # pylint: disable=E1101
).run_id

# Loop until the driver is disconnected
registered_nodes: Dict[int, DriverClientProxy] = {}
Expand All @@ -181,7 +183,7 @@ def update_client_manager(
if driver.stub is None:
break
get_nodes_res = driver.get_nodes(
req=driver_pb2.GetNodesRequest(run_id=run_id)
req=driver_pb2.GetNodesRequest(run_id=run_id) # pylint: disable=E1101
)
all_node_ids = {node.node_id for node in get_nodes_res.nodes}
dead_nodes = set(registered_nodes).difference(all_node_ids)
Expand Down
7 changes: 5 additions & 2 deletions src/py/flwr/driver/app_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@
from unittest.mock import MagicMock

from flwr.driver.app import update_client_manager
from flwr.proto.driver_pb2 import CreateRunResponse, GetNodesResponse
from flwr.proto.node_pb2 import Node
from flwr.proto.driver_pb2 import ( # pylint: disable=E0611
CreateRunResponse,
GetNodesResponse,
)
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.server.client_manager import SimpleClientManager


Expand Down
6 changes: 3 additions & 3 deletions src/py/flwr/driver/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
from typing import Iterable, List, Optional, Tuple

from flwr.driver.grpc_driver import DEFAULT_SERVER_ADDRESS_DRIVER, GrpcDriver
from flwr.proto.driver_pb2 import (
from flwr.proto.driver_pb2 import ( # pylint: disable=E0611
CreateRunRequest,
GetNodesRequest,
PullTaskResRequest,
PushTaskInsRequest,
)
from flwr.proto.node_pb2 import Node
from flwr.proto.task_pb2 import TaskIns, TaskRes
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611


class Driver:
Expand Down
Loading