refactor(integration): replace pytest.skip with pytest markers (#798)

Closes: #SYNC-4490

Makefile

@@ -2,7 +2,7 @@ SHELL := /bin/sh
 CARGO = cargo
 TESTS_DIR := tests
 TEST_RESULTS_DIR ?= workspace/test-results
-PYTEST_ARGS ?=
+PYTEST_ARGS ?= $(if $(SKIP_SENTRY),-m "not sentry") $(if $(TEST_STUB),,-m "not stub") # Stub tests do not work in CI
 INTEGRATION_TEST_FILE := $(TESTS_DIR)/integration/test_integration_all_rust.py
 LOAD_TEST_DIR := $(TESTS_DIR)/load
 POETRY := poetry --directory $(TESTS_DIR)
@@ -29,15 +29,15 @@ upgrade:
 
 integration-test-legacy:
 	$(POETRY) -V
-	$(POETRY) install --without dev,load --no-root
+	$(POETRY) install --without dev,load
 	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
 		--junit-xml=$(TEST_RESULTS_DIR)/integration_test_legacy_results.xml \
 		-v $(PYTEST_ARGS)
 
 integration-test:
 	$(POETRY) -V
-	$(POETRY) install --without dev,load --no-root
-		$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
+	$(POETRY) install --without dev,load
+	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
 		--junit-xml=$(TEST_RESULTS_DIR)/integration_test_results.xml \
 		-v $(PYTEST_ARGS)
 
@@ -72,7 +72,7 @@ pydocstyle: $(INSTALL_STAMP)  ##  Run pydocstyle
 
 lint:
 	$(POETRY) -V
-	$(POETRY) install --no-root
+	$(POETRY) install
 	$(POETRY) run isort --sp $(PYPROJECT_TOML) -c $(TESTS_DIR)
 	$(POETRY) run black --quiet --diff --config $(PYPROJECT_TOML) --check $(TESTS_DIR)
 	$(POETRY) run flake8 --config $(FLAKE8_CONFIG) $(TESTS_DIR)

docs/src/testing.md

@@ -84,6 +84,15 @@ You can alter the verbosity and logging output by adding command line flags to t
 
 The test output is then emitted in your terminal instance. This includes the name of the tests, whether they pass or fail and any exceptions that are triggered during the test run.
 
+The integration tests make use of [pytest markers][pytest_markers] for filtering tests. These can be
+used with the `-m` pytest option, or can be used through the following environment variables and
+`integration-test` make command.
+
+| ENVIRONMENT VARIABLE | RELATED MARKER | DESCRIPTION                                                       |
+|----------------------|----------------|-------------------------------------------------------------------|
+| SKIP_SENTRY          | sentry         | If set will exclude all tests marked with `sentry` from execution |
+| TEST_STUB            | stub           | If set will include all tests marked with `stub` in execution     |
+
 Integration tests in CI will be triggered automatically whenever a commit is pushed to a branch as a part of the CI PR workflow.
 
 ### Debugging
@@ -135,4 +144,5 @@ For more details see the [README.md][load_tests_docs] file in the `tests/load` d
 [integration_tests]: https://github.com/mozilla-services/autopush-rs/tree/master/tests/integration
 [integration_tests_docs]: ./testing.md#integration-tests
 [load_tests]: https://github.com/mozilla-services/autopush-rs/tree/master/tests/load
-[load_tests_docs]: https://github.com/mozilla-services/autopush-rs/blob/master/tests/load/README.md
+[load_tests_docs]: https://github.com/mozilla-services/autopush-rs/blob/master/tests/load/README.md
+[pytest_markers]: https://docs.pytest.org/en/stable/example/markers.html

tests/integration/test_integration_all_rust.py

@@ -634,12 +634,12 @@ async def fixture_registered_test_client(ws_config) -> AsyncGenerator:
     log.debug("🐍 Test Client Disconnected")
 
 
+@pytest.mark.sentry
+# Selectively include or exclude this test based on `mark` See
+# https://docs.pytest.org/en/7.1.x/example/markers.html
 @pytest.mark.parametrize("fixture_max_conn_logs", [4], indirect=True)
 async def test_sentry_output_autoconnect(test_client: AsyncPushTestClient) -> None:
     """Test sentry output for autoconnect."""
-    if os.getenv("SKIP_SENTRY"):
-        log.debug("Skipping test_sentry_output_autoconnect")
-        pytest.skip("Skipping test_sentry_output_autoconnect")
     # Ensure bad data doesn't throw errors
     await test_client.connect()
     await test_client.hello()
@@ -665,12 +665,10 @@ async def test_sentry_output_autoconnect(test_client: AsyncPushTestClient) -> No
     assert event1["exception"]["values"][0]["value"] == "LogCheck"
 
 
+@pytest.mark.sentry
 @pytest.mark.parametrize("fixture_max_endpoint_logs", [1], indirect=True)
 async def test_sentry_output_autoendpoint(registered_test_client: AsyncPushTestClient) -> None:
     """Test sentry output for autoendpoint."""
-    if os.getenv("SKIP_SENTRY"):
-        log.debug("Skipping test_sentry_output_autoendpoint")
-        pytest.skip("Skipping test_sentry_output_autoendpoint")
     endpoint = registered_test_client.get_host_client_endpoint()
     await registered_test_client.disconnect()
     async with httpx.AsyncClient() as httpx_client:
@@ -692,13 +690,10 @@ async def test_sentry_output_autoendpoint(registered_test_client: AsyncPushTestC
     assert sorted(values) == ["ERROR:Success", "LogCheck"]
 
 
+@pytest.mark.sentry
 @pytest.mark.parametrize("fixture_max_conn_logs", [4], indirect=True)
 async def test_no_sentry_output(ws_url: str) -> None:
     """Test for no Sentry output."""
-    if os.getenv("SKIP_SENTRY"):
-        log.debug("Skipping test_no_sentry_output")
-        pytest.skip("Skipping test_no_sentry_output")
-
     ws_url = urlparse(ws_url)._replace(scheme="http").geturl()
 
     try:
@@ -1571,13 +1566,11 @@ async def test_broadcast_no_changes(test_client_broadcast: AsyncPushTestClient)
 # Stub Tests
 # these tests are only really useful for testing the stub system and are
 # not required for production CI testing.
+@pytest.mark.stub
 async def test_mobile_register_v1(test_client: AsyncPushTestClient) -> None:
     """Test that the mobile "hello" request returns an unsigned endpoint
     if no `key` is included in the body
     """
-    if not os.getenv("TEST_STUB"):
-        pytest.skip("Skipping stub test test_mobile_register_v1")
-
     endpoint = test_client.get_host_client_endpoint()
     async with httpx.AsyncClient() as httpx_client:
         resp = await httpx_client.request(
@@ -1595,12 +1588,11 @@ async def test_mobile_register_v1(test_client: AsyncPushTestClient) -> None:
         assert response.get("uaid") is not None
 
 
+@pytest.mark.stub
 async def test_mobile_register_v2(test_client: AsyncPushTestClient) -> None:
     """Test that a signed endpoint is returned if a valid VAPID public
     key is included in the body.
     """
-    if not os.getenv("TEST_STUB"):
-        pytest.skip("Skipping stub test test_mobile_register_v2")
     vapid_pub = (
         "BBO5r087l4d3kxx9INyRenewaA5WOWiaSFqy77UXN7ZRVxr3gNtyWeP"
         "CjUbOerY1xUUcUFCtVoT5vdElIxTLlCc"

tests/pyproject.toml

@@ -87,4 +87,9 @@ requires = ["poetry-core>=1.9.1"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.pytest.ini_options]
-asyncio_mode = "auto"
+asyncio_mode = "auto"
+# Pytest marker documentation: https://docs.pytest.org/en/7.1.x/example/markers.html
+markers = [
+    "stub: mark a test for the stub system",
+    "sentry: mark a test for the sentry integration"
+]