Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

interoperability with asyncio (part 3): refactor with aiohttp #176

Merged
merged 35 commits into from
Jun 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
4a34c1c
aiohttp: refactor internals to use asyncio throughout the SDK
achille-roussel Jun 7, 2024
935ffdc
all tests pass using new APIs
achille-roussel Jun 11, 2024
42810bc
port http tests to generic dispatch test suite
achille-roussel Jun 12, 2024
731d96d
fix slow FastAPI tests
achille-roussel Jun 12, 2024
69e05bc
port some of the tests from TestCoroutine to TestCase + fix bugs
achille-roussel Jun 12, 2024
a674f07
fix compatibility with Python 3.8
achille-roussel Jun 12, 2024
8dfc03a
fix for Python 3.8: asyncio.Task is not a generic type
achille-roussel Jun 12, 2024
529823d
refactor: use composition, default registry, function service
achille-roussel Jun 14, 2024
6bfb086
remove unused variables
achille-roussel Jun 14, 2024
ecb9d33
enable nested function call test
achille-roussel Jun 14, 2024
d9e73d4
fix formatting check for registry endpoint
achille-roussel Jun 14, 2024
b894337
fix PrimitiveFunction.__call__, the method cannot be async
achille-roussel Jun 14, 2024
b4e2db4
more fixes after rebase
achille-roussel Jun 14, 2024
3d0b0bb
cleanup
achille-roussel Jun 15, 2024
f0a54e2
cleanup
achille-roussel Jun 15, 2024
60c2312
try fixing compat with 3.8
achille-roussel Jun 17, 2024
c053298
fix python version check
achille-roussel Jun 17, 2024
d6fa555
emulate wait capability
achille-roussel Jun 17, 2024
ccc4ab8
rewrite examples to use dispatch.run
achille-roussel Jun 17, 2024
2803551
fix emulation of call results (don't end on temporary errors)
achille-roussel Jun 17, 2024
0c186a3
handle client connection errors
achille-roussel Jun 17, 2024
6799ab3
fix formatting
achille-roussel Jun 17, 2024
a406d2a
remove type annotations from examples so they are compatible with Pyt…
achille-roussel Jun 17, 2024
d5f5848
use context varaibles instead of thread locals
achille-roussel Jun 18, 2024
0918918
update README
achille-roussel Jun 18, 2024
d016221
fix running examples as tests
achille-roussel Jun 18, 2024
cf363d0
switch pytest.mark.asyncio position because maybe that's why the test…
achille-roussel Jun 18, 2024
debe5d8
fix tests for Python 3.9
achille-roussel Jun 18, 2024
8487159
fix formatting
achille-roussel Jun 18, 2024
672b1a1
re-enable the batch API
achille-roussel Jun 18, 2024
fea85a8
remove unused files
achille-roussel Jun 18, 2024
8394ee5
fix formatting
achille-roussel Jun 18, 2024
d448489
add documentation
achille-roussel Jun 18, 2024
e340938
support both asyncio and blocking modes with different abstractions
achille-roussel Jun 18, 2024
41c6ad3
Merge pull request #178 from dispatchrun/asyncio-or-blocking
achille-roussel Jun 19, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ import dispatch
def greet(msg: str):
print(f"Hello, ${msg}!")

dispatch.run(lambda: greet.dispatch('World'))
dispatch.run(greet('World'))
```

Obviously, this is just an example, a real application would perform much more
Expand Down
29 changes: 29 additions & 0 deletions examples/auto_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import random

import requests

import dispatch
import dispatch.integrations.requests

rng = random.Random(2)


def third_party_api_call(x):
# Simulate a third-party API call that fails.
print(f"Simulating third-party API call with {x}")
if x < 3:
print("RAISE EXCEPTION")
raise requests.RequestException("Simulated failure")
else:
return "SUCCESS"


# Use the `dispatch.function` decorator to declare a stateful function.
@dispatch.function
def auto_retry():
x = rng.randint(0, 5)
return third_party_api_call(x)


if __name__ == "__main__":
print(dispatch.run(auto_retry()))
Empty file removed examples/auto_retry/__init__.py
Empty file.
64 changes: 0 additions & 64 deletions examples/auto_retry/app.py

This file was deleted.

51 changes: 0 additions & 51 deletions examples/auto_retry/test_app.py

This file was deleted.

41 changes: 9 additions & 32 deletions examples/fanout/fanout.py → examples/fanout.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,10 @@
"""Fan-out example using the SDK gather feature

This example demonstrates how to use the SDK to fan-out multiple requests.

Run with:

uvicorn fanout:app


You will observe that the get_repo_info calls are executed in parallel.

"""

import httpx
from fastapi import FastAPI

from dispatch import gather
from dispatch.fastapi import Dispatch

app = FastAPI()

dispatch = Dispatch(app)
import dispatch


@dispatch.function
async def get_repo(repo_owner: str, repo_name: str):
def get_repo(repo_owner: str, repo_name: str):
url = f"https://api.github.com/repos/{repo_owner}/{repo_name}"
api_response = httpx.get(url)
api_response.raise_for_status()
Expand All @@ -32,7 +13,7 @@ async def get_repo(repo_owner: str, repo_name: str):


@dispatch.function
async def get_stargazers(repo_info):
def get_stargazers(repo_info):
url = repo_info["stargazers_url"]
response = httpx.get(url)
response.raise_for_status()
Expand All @@ -42,7 +23,7 @@ async def get_stargazers(repo_info):

@dispatch.function
async def reduce_stargazers(repos):
result = await gather(*[get_stargazers(repo) for repo in repos])
result = await dispatch.gather(*[get_stargazers(repo) for repo in repos])
reduced_stars = set()
for repo in result:
for stars in repo:
Expand All @@ -52,18 +33,14 @@ async def reduce_stargazers(repos):

@dispatch.function
async def fanout():
# Using gather, we fan-out the four following requests.
repos = await gather(
# Using gather, we fan-out the following requests:
repos = await dispatch.gather(
get_repo("dispatchrun", "coroutine"),
get_repo("dispatchrun", "dispatch-py"),
get_repo("dispatchrun", "wzprof"),
)

stars = await reduce_stargazers(repos)
print("Total stars:", len(stars))
return await reduce_stargazers(repos)


@app.get("/")
def root():
fanout.dispatch()
return "OK"
if __name__ == "__main__":
print(dispatch.run(fanout()))
Empty file removed examples/fanout/__init__.py
Empty file.
19 changes: 0 additions & 19 deletions examples/fanout/test_fanout.py

This file was deleted.

19 changes: 19 additions & 0 deletions examples/getting_started.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import requests

import dispatch


@dispatch.function
def publish(url, payload):
r = requests.post(url, data=payload)
r.raise_for_status()
return r.text


@dispatch.function
async def getting_started():
return await publish("https://httpstat.us/200", {"hello": "world"})


if __name__ == "__main__":
print(dispatch.run(getting_started()))
Empty file.
85 changes: 0 additions & 85 deletions examples/getting_started/app.py

This file was deleted.

40 changes: 0 additions & 40 deletions examples/getting_started/test_app.py

This file was deleted.

Loading