Timing Demo

python/ray/experimental/serve/examples/demo_profile.py

@@ -0,0 +1,39 @@
+"""
+Let this file run.
+And in a new terminal, run ApacheBench
+   ab -c 20 -n 60 http://127.0.0.1:8000/sleep
+
+Ploting code
+
+import pandas as pd
+import matplotlib.pyplot as plt
+df = pd.read_json('/tmp/serve_profile.jsonl', lines=True)
+for i, row in df.iterrows():
+    plt.plot([row['start'], row['end']], [i,i])
+plt.xlabel("UNIX Timestamp")
+plt.ylabel("Query ID")
+plt.title("Handling ab -c 20 -n 60 http://127.0.0.1:8000/sleep")
+plt.savefig("profile.png")
+"""
+
+import time
+
+import requests
+
+from ray.experimental import serve
+from ray.experimental.serve.utils import pformat_color_json
+
+
+def echo(flask_request):
+    time.sleep(0.1) # sleep to 0.1 seconds
+    return "done"
+
+
+serve.init(blocking=True)
+serve.create_endpoint("my_endpoint", "/sleep", blocking=True)
+serve.create_backend(echo, "echo:v1")
+serve.link("my_endpoint", "echo:v1")
+serve.scale("echo:v1", 4)
+
+while True:
+    time.sleep(1000)

python/ray/experimental/serve/server.py

@@ -1,5 +1,7 @@
 import asyncio
 import json
+import time
+import os
 
 import uvicorn
 
@@ -73,6 +75,8 @@ def __init__(self, kv_store_actor_handle, router_handle):
 
         self.route_checker_should_shutdown = False
 
+        self.profile_file = open(os.environ.get("SERVE_PROFILE_PATH", "/tmp/serve_profile.jsonl"),'w')
+
     async def route_checker(self, interval):
         while True:
             if self.route_checker_should_shutdown:
@@ -138,6 +142,7 @@ async def __call__(self, scope, receive, send):
         endpoint_name = self.route_table[current_path]
         http_body_bytes = await self.receive_http_body(scope, receive, send)
 
+        request_sent_time = time.time()
         result_object_id_bytes = await as_future(
             self.router.enqueue_request.remote(
                 service=endpoint_name,
@@ -146,6 +151,11 @@ async def __call__(self, scope, receive, send):
                 request_context=TaskContext.Web))
 
         result = await as_future(ray.ObjectID(result_object_id_bytes))
+        result_received_time = time.time()
+        self.profile_file.write(json.dumps({"start": request_sent_time, "end": result_received_time}))
+        self.profile_file.write("\n")
+        self.profile_file.flush()
+
 
         if isinstance(result, ray.exceptions.RayTaskError):
             await JSONResponse({