feat: Implement performance benchmarking (#443)

* feat: Implement performance benchmarking

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/codspeed.yml

@@ -0,0 +1,37 @@
+name: codspeed-benchmarks
+
+on:
+  push:
+    branches:
+      - "main" # or "master"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.7"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r robyn/test-requirements.txt
+      - name: Add macos target
+        if: matrix.os == 'macos'
+        run: rustup target add aarch64-apple-darwin
+      - name: Setup Rust part of the project
+        run: |
+          maturin build -i python --universal2 --out dist
+          pip install --no-index --find-links=dist/ robyn
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v1
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: pytest integration_tests --codspeed

integration_tests/test_app.py

@@ -2,19 +2,24 @@
 from robyn.events import Events
 from robyn.types import Header
 
+import pytest
 
+
+@pytest.mark.benchmark
 def test_add_request_header():
     app = Robyn(__file__)
     app.add_request_header("server", "robyn")
     assert app.request_headers == [Header(key="server", val="robyn")]
 
 
+@pytest.mark.benchmark
 def test_add_response_header():
     app = Robyn(__file__)
     app.add_response_header("content-type", "application/json")
     assert app.response_headers == [Header(key="content-type", val="application/json")]
 
 
+@pytest.mark.benchmark
 def test_lifecycle_handlers():
     def mock_startup_handler():
         pass

integration_tests/test_base_url.py

@@ -1,23 +1,27 @@
 import os
+import pytest
 
 import requests
 
 from helpers.network_helpers import get_network_host
 
 
+@pytest.mark.benchmark
 def test_default_url_index_request(default_session):
     BASE_URL = "http://127.0.0.1:8080"
     res = requests.get(f"{BASE_URL}")
     assert res.status_code == 200
 
 
+@pytest.mark.benchmark
 def test_local_index_request(session):
     BASE_URL = "http://127.0.0.1:8080"
     res = requests.get(f"{BASE_URL}")
     assert os.getenv("ROBYN_URL") == "127.0.0.1"
     assert res.status_code == 200
 
 
+@pytest.mark.benchmark
 def test_global_index_request(global_session):
     host = get_network_host()
     BASE_URL = f"http://{host}:8080"
@@ -26,6 +30,7 @@ def test_global_index_request(global_session):
     assert res.status_code == 200
 
 
+@pytest.mark.benchmark
 def test_dev_index_request(dev_session):
     BASE_URL = "http://127.0.0.1:8081"
     res = requests.get(f"{BASE_URL}")

integration_tests/test_basic_routes.py

@@ -10,6 +10,7 @@
 from helpers.http_methods_helpers import get
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize(
     "route,expected_text,expected_header_key,expected_header_value",
     [
@@ -46,6 +47,7 @@ def test_basic_get(
         assert res.headers[expected_header_key] == expected_header_value
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize(
     "route, expected_json",
     [

integration_tests/test_binary_output.py

@@ -1,29 +1,34 @@
 import requests
+import pytest
 
 BASE_URL = "http://127.0.0.1:8080"
 
 
+@pytest.mark.benchmark
 def test_binary_output_sync(session):
     r = requests.get(f"{BASE_URL}/binary_output_sync")
     assert r.status_code == 200
     assert r.headers["Content-Type"] == "application/octet-stream"
     assert r.text == "OK"
 
 
+@pytest.mark.benchmark
 def test_binary_output_response_sync(session):
     r = requests.get(f"{BASE_URL}/binary_output_response_sync")
     assert r.status_code == 200
     assert r.headers["Content-Type"] == "application/octet-stream"
     assert r.text == "OK"
 
 
+@pytest.mark.benchmark
 def test_binary_output_async(session):
     r = requests.get(f"{BASE_URL}/binary_output_async")
     assert r.status_code == 200
     assert r.headers["Content-Type"] == "application/octet-stream"
     assert r.text == "OK"
 
 
+@pytest.mark.benchmark
 def test_binary_output_response_async(session):
     r = requests.get(f"{BASE_URL}/binary_output_response_async")
     assert r.status_code == 200

integration_tests/test_delete_requests.py

@@ -2,6 +2,7 @@
 from helpers.http_methods_helpers import delete
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_delete(function_type: str, session):
     res = delete(f"/{function_type}/dict")
@@ -10,6 +11,7 @@ def test_delete(function_type: str, session):
     assert res.headers[function_type] == "dict"
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_delete_with_param(function_type: str, session):
     res = delete(f"/{function_type}/body", data={"hello": "world"})

integration_tests/test_env_populator.py

@@ -1,10 +1,12 @@
 import os
 import pathlib
+import pytest
 
 from robyn.env_populator import load_vars, parser
 
 
 # this tests if a connection can be made to the server with the correct port imported from the env file
+@pytest.mark.benchmark
 def test_env_population(test_session, env_file):
     path = pathlib.Path(__file__).parent
     env_path = path / "robyn.env"

integration_tests/test_file_download.py

@@ -2,6 +2,7 @@
 from helpers.http_methods_helpers import get
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_file_download(function_type: str, session):
     r = get(f"/{function_type}/file/download")

integration_tests/test_get_requests.py

@@ -4,6 +4,7 @@
 from helpers.http_methods_helpers import get
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_param(function_type: str, session):
     r = get(f"/{function_type}/param/1")
@@ -12,6 +13,7 @@ def test_param(function_type: str, session):
     assert r.text == "12345"
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_serve_html(function_type: str, session):
     def check_response(r: Response):
@@ -21,6 +23,7 @@ def check_response(r: Response):
     check_response(get(f"/{function_type}/serve/html"))
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_template(function_type: str, session):
     def check_response(r: Response):
@@ -31,6 +34,7 @@ def check_response(r: Response):
     check_response(get(f"/{function_type}/template"))
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_queries(function_type: str, session):
     r = get(f"/{function_type}/queries?hello=robyn")

integration_tests/test_middlewares.py

@@ -3,6 +3,7 @@
 from helpers.http_methods_helpers import get
 
 
+@pytest.mark.benchmark
 @pytest.mark.skip(reason="Fix middleware request headers modification")
 def test_middlewares(session):
     r = get("/")

integration_tests/test_patch_requests.py

@@ -2,6 +2,7 @@
 from helpers.http_methods_helpers import patch
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_patch(function_type: str, session):
     res = patch(f"/{function_type}/dict")
@@ -10,6 +11,7 @@ def test_patch(function_type: str, session):
     assert res.headers[function_type] == "dict"
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_patch_with_param(function_type: str, session):
     res = patch(f"/{function_type}/body", data={"hello": "world"})

integration_tests/test_post_requests.py

@@ -2,6 +2,7 @@
 from helpers.http_methods_helpers import post
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_post(function_type: str, session):
     res = post(f"/{function_type}/dict")
@@ -10,6 +11,7 @@ def test_post(function_type: str, session):
     assert res.headers[function_type] == "dict"
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_post_with_param(function_type: str, session):
     res = post(f"/{function_type}/body", data={"hello": "world"})

integration_tests/test_put_requests.py

@@ -2,6 +2,7 @@
 from helpers.http_methods_helpers import put
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_put(function_type: str, session):
     res = put(f"/{function_type}/dict")
@@ -10,6 +11,7 @@ def test_put(function_type: str, session):
     assert res.headers[function_type] == "dict"
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_put_with_param(function_type: str, session):
     res = put(f"/{function_type}/body", data={"hello": "world"})

integration_tests/test_status_code.py

@@ -2,19 +2,23 @@
 from helpers.http_methods_helpers import get
 
 
+@pytest.mark.benchmark
 def test_404_status_code(session):
     get("/404", expected_status_code=404)
 
 
+@pytest.mark.benchmark
 def test_404_not_found(session):
     r = get("/real/404", expected_status_code=404)
     assert r.text == "Not found"
 
 
+@pytest.mark.benchmark
 def test_202_status_code(session):
     get("/202", expected_status_code=202)
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("function_type", ["sync", "async"])
 def test_sync_500_internal_server_error(function_type: str, session):
     get(f"/{function_type}/raise", expected_status_code=500)

integration_tests/test_views.py

@@ -1,41 +1,50 @@
 from helpers.http_methods_helpers import get, post
+import pytest
 
 
+@pytest.mark.benchmark
 def test_get_sync_view(session):
     r = get("/sync/view")
     assert r.text == "Hello, world!"
 
 
+@pytest.mark.benchmark
 def test_post_sync_view(session):
     r = post("/sync/view", data={"name": "John"})
     assert "John" in r.text
 
 
+@pytest.mark.benchmark
 def test_get_sync_decorator_view(session):
     r = get("/sync/view/decorator")
     assert r.text == "Hello, world!"
 
 
+@pytest.mark.benchmark
 def test_post_sync_decorator_view(session):
     r = post("/sync/view/decorator", data={"name": "John"})
     assert "John" in r.text
 
 
+@pytest.mark.benchmark
 def test_get_async_view(session):
     r = get("/async/view")
     assert r.text == "Hello, world!"
 
 
+@pytest.mark.benchmark
 def test_post_async_view(session):
     r = post("/async/view", data={"name": "John"})
     assert "John" in r.text
 
 
+@pytest.mark.benchmark
 def test_get_async_decorator_view(session):
     r = get("/async/view/decorator")
     assert r.text == "Hello, world!"
 
 
+@pytest.mark.benchmark
 def test_post_async_decorator_view(session):
     r = post("/async/view/decorator", data={"name": "John"})
     assert "John" in r.text

integration_tests/test_web_sockets.py

@@ -1,8 +1,10 @@
 from websocket import create_connection
+import pytest
 
 BASE_URL = "ws://127.0.0.1:8080"
 
 
+@pytest.mark.benchmark
 def test_web_socket(session):
     ws = create_connection(f"{BASE_URL}/web_socket")
     assert ws.recv() == "Hello world, from ws"

robyn/test-requirements.txt

@@ -2,3 +2,4 @@
 requests==2.28.2
 pytest==7.2.1
 websocket-client==1.5.0
+pytest-codspeed