From b2c398e5a42bef56c5b19169a5f95fa1a2a5bb2b Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 11:42:06 -0700 Subject: [PATCH 01/10] Create doc code for websockets Signed-off-by: Shreyas Krishnaswamy --- doc/BUILD | 4 +-- .../doc_code/{ => http_guide}/http_guide.py | 0 .../{ => http_guide}/streaming_example.py | 0 .../serve/doc_code/http_guide/websockets.py | 36 +++++++++++++++++++ doc/source/serve/http-guide.md | 22 +++++++----- 5 files changed, 52 insertions(+), 10 deletions(-) rename doc/source/serve/doc_code/{ => http_guide}/http_guide.py (100%) rename doc/source/serve/doc_code/{ => http_guide}/streaming_example.py (100%) create mode 100644 doc/source/serve/doc_code/http_guide/websockets.py diff --git a/doc/BUILD b/doc/BUILD index 1684f455e492b..013e2858f0037 100644 --- a/doc/BUILD +++ b/doc/BUILD @@ -133,7 +133,7 @@ py_test_run_all_subdirectory( "source/serve/doc_code/distilbert.py", "source/serve/doc_code/stable_diffusion.py", "source/serve/doc_code/object_detection.py", - "source/serve/doc_code/streaming_example.py", + "source/serve/doc_code/http_guide/streaming_example.py", ], extra_srcs = [], tags = ["exclusive", "team:serve"], @@ -141,7 +141,7 @@ py_test_run_all_subdirectory( py_test_run_all_subdirectory( size = "medium", - include = ["source/serve/doc_code/streaming_example.py"], + include = ["source/serve/doc_code/http_guide/streaming_example.py"], exclude = [], extra_srcs = [], tags = ["exclusive", "team:serve"], diff --git a/doc/source/serve/doc_code/http_guide.py b/doc/source/serve/doc_code/http_guide/http_guide.py similarity index 100% rename from doc/source/serve/doc_code/http_guide.py rename to doc/source/serve/doc_code/http_guide/http_guide.py diff --git a/doc/source/serve/doc_code/streaming_example.py b/doc/source/serve/doc_code/http_guide/streaming_example.py similarity index 100% rename from doc/source/serve/doc_code/streaming_example.py rename to doc/source/serve/doc_code/http_guide/streaming_example.py diff --git a/doc/source/serve/doc_code/http_guide/websockets.py b/doc/source/serve/doc_code/http_guide/websockets.py new file mode 100644 index 0000000000000..57caee4858dc0 --- /dev/null +++ b/doc/source/serve/doc_code/http_guide/websockets.py @@ -0,0 +1,36 @@ +# __websocket_serve_app_start__ +from fastapi import FastAPI, WebSocket, WebSocketDisconnect + +from ray import serve + + +app = FastAPI() + +@serve.deployment +@serve.ingress(app) +class EchoServer: + + @app.websocket("/") + async def ws_handler(self, ws: WebSocket): + await ws.accept() + + try: + while True: + text = await ws.receive_text() + await ws.send_text(text) + except WebSocketDisconnect: + print("Client disconnected.") + +serve_app = serve.run(EchoServer.bind()) +# __websocket_serve_app_end__ + +# __websocket_serve_client_start__ +from websockets.sync.client import connect + +with connect("ws://localhost:8000") as websocket: + websocket.send("Eureka!") + assert websocket.recv() == "Eureka!" + + websocket.send("I've found it!") + assert websocket.recv().decode("utf-8") == "I've found it!" +# __websocket_serve_client_end__ diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index 7e9cee7adfc6c..8eac1f9b4fc37 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -21,7 +21,7 @@ Considering your use case, you can choose the right level of abstraction: ## Calling Deployments via HTTP When you deploy a Serve application, the [ingress deployment](serve-key-concepts-ingress-deployment) (the one passed to `serve.run`) will be exposed over HTTP. -```{literalinclude} doc_code/http_guide.py +```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_starlette__ :end-before: __end_starlette__ :language: python @@ -31,7 +31,7 @@ Requests to the Serve HTTP server at `/` are routed to the deployment's `__call_ Often for ML models, you just need the API to accept a `numpy` array. You can use Serve's `DAGDriver` to simplify the request parsing. -```{literalinclude} doc_code/http_guide.py +```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_dagdriver__ :end-before: __end_dagdriver__ :language: python @@ -46,7 +46,7 @@ Serve provides a library of HTTP adapters to help you avoid boilerplate code. Th If you want to define more complex HTTP handling logic, Serve integrates with [FastAPI](https://fastapi.tiangolo.com/). This allows you to define a Serve deployment using the {mod}`@serve.ingress ` decorator that wraps a FastAPI app with its full range of features. The most basic example of this is shown below, but for more details on all that FastAPI has to offer such as variable routes, automatic type validation, dependency injection (e.g., for database connections), and more, please check out [their documentation](https://fastapi.tiangolo.com/). -```{literalinclude} doc_code/http_guide.py +```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_fastapi__ :end-before: __end_fastapi__ :language: python @@ -54,7 +54,7 @@ If you want to define more complex HTTP handling logic, Serve integrates with [F Now if you send a request to `/hello`, this will be routed to the `root` method of our deployment. We can also easily leverage FastAPI to define multiple routes with different HTTP methods: -```{literalinclude} doc_code/http_guide.py +```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_fastapi_multi_routes__ :end-before: __end_fastapi_multi_routes__ :language: python @@ -62,7 +62,7 @@ Now if you send a request to `/hello`, this will be routed to the `root` method You can also pass in an existing FastAPI app to a deployment to serve it as-is: -```{literalinclude} doc_code/http_guide.py +```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_byo_fastapi__ :end-before: __end_byo_fastapi__ :language: python @@ -71,10 +71,16 @@ You can also pass in an existing FastAPI app to a deployment to serve it as-is: This is useful for scaling out an existing FastAPI app with no modifications necessary. Existing middlewares, **automatic OpenAPI documentation generation**, and other advanced FastAPI features should work as-is. -```{note} -Serve currently does not support WebSockets. If you have a use case that requires it, please [let us know](https://github.com/ray-project/ray/issues/new/choose)! +### Websockets + +```{warning} +Support for WebSockets is experimental. To enable this feature, set `RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1` on the cluster before starting Ray. If you encounter any issues, [file an issue on GitHub](https://github.com/ray-project/ray/issues/new/choose). ``` +Serve offers websocket support via FastAPI: + + + (serve-http-streaming-response)= ## Streaming Responses @@ -93,7 +99,7 @@ The code below defines a Serve application that incrementally streams numbers up The client-side code is also updated to handle the streaming outputs. This code uses the `stream=True` option to the [requests](https://requests.readthedocs.io/en/latest/user/advanced/#streaming-requests) library. -```{literalinclude} doc_code/streaming_example.py +```{literalinclude} doc_code/http_guide/streaming_example.py :start-after: __begin_example__ :end-before: __end_example__ :language: python From 8338fbc6041c41814d32c37aa23891fffaac0634 Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 11:42:39 -0700 Subject: [PATCH 02/10] Add noqa Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/doc_code/http_guide/websockets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/serve/doc_code/http_guide/websockets.py b/doc/source/serve/doc_code/http_guide/websockets.py index 57caee4858dc0..9f12ea917c315 100644 --- a/doc/source/serve/doc_code/http_guide/websockets.py +++ b/doc/source/serve/doc_code/http_guide/websockets.py @@ -1,3 +1,5 @@ +# flake8: noqa + # __websocket_serve_app_start__ from fastapi import FastAPI, WebSocket, WebSocketDisconnect @@ -6,10 +8,10 @@ app = FastAPI() + @serve.deployment @serve.ingress(app) class EchoServer: - @app.websocket("/") async def ws_handler(self, ws: WebSocket): await ws.accept() @@ -21,6 +23,7 @@ async def ws_handler(self, ws: WebSocket): except WebSocketDisconnect: print("Client disconnected.") + serve_app = serve.run(EchoServer.bind()) # __websocket_serve_app_end__ From 17b184372ff6fb65a7057b9850cd7edc197db0ac Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 11:59:24 -0700 Subject: [PATCH 03/10] Change ws_handler to echo Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/doc_code/http_guide/websockets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/serve/doc_code/http_guide/websockets.py b/doc/source/serve/doc_code/http_guide/websockets.py index 9f12ea917c315..c621f9a24050f 100644 --- a/doc/source/serve/doc_code/http_guide/websockets.py +++ b/doc/source/serve/doc_code/http_guide/websockets.py @@ -13,7 +13,7 @@ @serve.ingress(app) class EchoServer: @app.websocket("/") - async def ws_handler(self, ws: WebSocket): + async def echo(self, ws: WebSocket): await ws.accept() try: @@ -23,7 +23,6 @@ async def ws_handler(self, ws: WebSocket): except WebSocketDisconnect: print("Client disconnected.") - serve_app = serve.run(EchoServer.bind()) # __websocket_serve_app_end__ From 2ed2ab9ad15947df1fe30ce670b074d44be86cb5 Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 12:06:17 -0700 Subject: [PATCH 04/10] Fix log message Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/http-guide.md | 1 + python/ray/serve/_private/http_state.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index 8eac1f9b4fc37..6ee380cd9f6a6 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -81,6 +81,7 @@ Serve offers websocket support via FastAPI: + (serve-http-streaming-response)= ## Streaming Responses diff --git a/python/ray/serve/_private/http_state.py b/python/ray/serve/_private/http_state.py index 127febbf07699..fe91c7cda40e7 100644 --- a/python/ray/serve/_private/http_state.py +++ b/python/ray/serve/_private/http_state.py @@ -382,8 +382,8 @@ def _stop_proxies_if_needed(self) -> bool: to_stop.append(node_id) elif proxy_state.status == HTTPProxyStatus.UNHEALTHY: logger.info( - f"HTTP proxy on node '{node_id}' UNHEALTHY. Shut down the unhealthy" - " proxy and restart a new one." + f"HTTP proxy on node '{node_id}' UNHEALTHY. Shutting down " + "the unhealthy proxy and starting a new one." ) to_stop.append(node_id) From d28f4253549e56ee9c07c2c7648ec0429bf4569d Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 15:52:41 -0700 Subject: [PATCH 05/10] Rename websockets.py to websockets_example.py Signed-off-by: Shreyas Krishnaswamy --- doc/BUILD | 6 +++++- .../http_guide/{websockets.py => websockets_example.py} | 0 2 files changed, 5 insertions(+), 1 deletion(-) rename doc/source/serve/doc_code/http_guide/{websockets.py => websockets_example.py} (100%) diff --git a/doc/BUILD b/doc/BUILD index 013e2858f0037..256ff965e96ae 100644 --- a/doc/BUILD +++ b/doc/BUILD @@ -134,6 +134,7 @@ py_test_run_all_subdirectory( "source/serve/doc_code/stable_diffusion.py", "source/serve/doc_code/object_detection.py", "source/serve/doc_code/http_guide/streaming_example.py", + "source/serve/doc_code/http_guide/websockets_example.py", ], extra_srcs = [], tags = ["exclusive", "team:serve"], @@ -141,7 +142,10 @@ py_test_run_all_subdirectory( py_test_run_all_subdirectory( size = "medium", - include = ["source/serve/doc_code/http_guide/streaming_example.py"], + include = [ + "source/serve/doc_code/http_guide/streaming_example.py", + "source/serve/doc_code/http_guide/websockets_example.py", + ], exclude = [], extra_srcs = [], tags = ["exclusive", "team:serve"], diff --git a/doc/source/serve/doc_code/http_guide/websockets.py b/doc/source/serve/doc_code/http_guide/websockets_example.py similarity index 100% rename from doc/source/serve/doc_code/http_guide/websockets.py rename to doc/source/serve/doc_code/http_guide/websockets_example.py From d81234a7b8f19cbe40adff98d586e6ec9061087e Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 15:59:11 -0700 Subject: [PATCH 06/10] Finish section about websockets Signed-off-by: Shreyas Krishnaswamy --- .../doc_code/http_guide/websockets_example.py | 3 ++- doc/source/serve/http-guide.md | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/doc/source/serve/doc_code/http_guide/websockets_example.py b/doc/source/serve/doc_code/http_guide/websockets_example.py index c621f9a24050f..6cabc6c11b05e 100644 --- a/doc/source/serve/doc_code/http_guide/websockets_example.py +++ b/doc/source/serve/doc_code/http_guide/websockets_example.py @@ -23,6 +23,7 @@ async def echo(self, ws: WebSocket): except WebSocketDisconnect: print("Client disconnected.") + serve_app = serve.run(EchoServer.bind()) # __websocket_serve_app_end__ @@ -34,5 +35,5 @@ async def echo(self, ws: WebSocket): assert websocket.recv() == "Eureka!" websocket.send("I've found it!") - assert websocket.recv().decode("utf-8") == "I've found it!" + assert websocket.recv() == "I've found it!" # __websocket_serve_client_end__ diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index 6ee380cd9f6a6..cc6c6c88150a7 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -79,8 +79,29 @@ Support for WebSockets is experimental. To enable this feature, set `RAY_SERVE_E Serve offers websocket support via FastAPI: +```{literalinclude} doc_code/http_guide/websockets_example.py +:start-after: __websocket_serve_app_start__ +:end-before: __websocket_serve_app_end__ +:language: python +``` + +Decorate the function that handles websocket requests using `@app.websocket`. Read more about FastAPI websockets in the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). + +:::{tip} +Install the `websockets` package to query the Serve deployment: + +``` +pip install websockets +``` +::: +Query the deployment using the `websockets` package: +```{literalinclude} doc_code/http_guide/websockets_example.py +:start-after: __websocket_serve_app_start__ +:end-before: __websocket_serve_app_end__ +:language: python +``` (serve-http-streaming-response)= ## Streaming Responses From 5527d1eede43e0bd5147065fedede4e6fe65858c Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Thu, 22 Jun 2023 16:03:51 -0700 Subject: [PATCH 07/10] Remove tip Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/http-guide.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index cc6c6c88150a7..c19e41a2bfff8 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -85,17 +85,9 @@ Serve offers websocket support via FastAPI: :language: python ``` -Decorate the function that handles websocket requests using `@app.websocket`. Read more about FastAPI websockets in the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). +Decorate the function that handles websocket requests with `@app.websocket`. Read more about FastAPI websockets in the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). -:::{tip} -Install the `websockets` package to query the Serve deployment: - -``` -pip install websockets -``` -::: - -Query the deployment using the `websockets` package: +Query the deployment using the `websockets` package (`pip install websockets`): ```{literalinclude} doc_code/http_guide/websockets_example.py :start-after: __websocket_serve_app_start__ From beeb5428df56b3c80f8d1698a569c681d03a3277 Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Fri, 23 Jun 2023 10:15:40 -0700 Subject: [PATCH 08/10] Fix second code snippet Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/http-guide.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index c19e41a2bfff8..b665571a94006 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -71,13 +71,13 @@ You can also pass in an existing FastAPI app to a deployment to serve it as-is: This is useful for scaling out an existing FastAPI app with no modifications necessary. Existing middlewares, **automatic OpenAPI documentation generation**, and other advanced FastAPI features should work as-is. -### Websockets +### WebSockets ```{warning} Support for WebSockets is experimental. To enable this feature, set `RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1` on the cluster before starting Ray. If you encounter any issues, [file an issue on GitHub](https://github.com/ray-project/ray/issues/new/choose). ``` -Serve offers websocket support via FastAPI: +Serve supports WebSockets via FastAPI: ```{literalinclude} doc_code/http_guide/websockets_example.py :start-after: __websocket_serve_app_start__ @@ -85,13 +85,13 @@ Serve offers websocket support via FastAPI: :language: python ``` -Decorate the function that handles websocket requests with `@app.websocket`. Read more about FastAPI websockets in the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). +Decorate the function that handles WebSocket requests with `@app.websocket`. Read more about FastAPI WebSockets in the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/websockets/). Query the deployment using the `websockets` package (`pip install websockets`): ```{literalinclude} doc_code/http_guide/websockets_example.py -:start-after: __websocket_serve_app_start__ -:end-before: __websocket_serve_app_end__ +:start-after: __websocket_serve_client_start__ +:end-before: __websocket_serve_client_end__ :language: python ``` From ca4b3c8b6b6d91752d8b6c9e861716c2515be648 Mon Sep 17 00:00:00 2001 From: Shreyas Krishnaswamy Date: Fri, 23 Jun 2023 12:24:57 -0700 Subject: [PATCH 09/10] Fix type hint Signed-off-by: Shreyas Krishnaswamy --- doc/source/serve/doc_code/batching_guide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/serve/doc_code/batching_guide.py b/doc/source/serve/doc_code/batching_guide.py index 2affce891245c..6a0fd978fb668 100644 --- a/doc/source/serve/doc_code/batching_guide.py +++ b/doc/source/serve/doc_code/batching_guide.py @@ -83,8 +83,8 @@ def __call__(self, request: Request) -> StreamingResponse: class StreamingResponder: @serve.batch(max_batch_size=5, batch_wait_timeout_s=0.1) async def generate_numbers( - self, max_list: List[Union[str, StopIteration]] - ) -> AsyncGenerator[List[int], None]: + self, max_list: List[str] + ) -> AsyncGenerator[List[Union[int, StopIteration]], None]: for i in range(max(max_list)): next_numbers = [] for requested_max in max_list: From a9e9fd6e9f899b320d4fdfcfb9b34130c77c742d Mon Sep 17 00:00:00 2001 From: shrekris-anyscale <92341594+shrekris-anyscale@users.noreply.github.com> Date: Fri, 23 Jun 2023 16:11:50 -0700 Subject: [PATCH 10/10] Update doc/source/serve/http-guide.md Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> Signed-off-by: shrekris-anyscale <92341594+shrekris-anyscale@users.noreply.github.com> --- doc/source/serve/http-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index b665571a94006..bd0307e636df7 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -19,7 +19,7 @@ Considering your use case, you can choose the right level of abstraction: (serve-http)= ## Calling Deployments via HTTP -When you deploy a Serve application, the [ingress deployment](serve-key-concepts-ingress-deployment) (the one passed to `serve.run`) will be exposed over HTTP. +When you deploy a Serve application, the [ingress deployment](serve-key-concepts-ingress-deployment) (the one passed to `serve.run`) is exposed over HTTP. ```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_starlette__