From 913ef9c29ec7ef9cbf700ce3f53ec3de27dabfc5 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 1 Sep 2021 16:01:13 +0200 Subject: [PATCH 1/5] Fix #838: check to create a task --- checks/taskcluster/create_task.py | 78 ++++++++++++++++++++ poucave/config.py | 1 + tests/checks/taskcluster/test_create_task.py | 42 +++++++++++ 3 files changed, 121 insertions(+) create mode 100644 checks/taskcluster/create_task.py create mode 100644 tests/checks/taskcluster/test_create_task.py diff --git a/checks/taskcluster/create_task.py b/checks/taskcluster/create_task.py new file mode 100644 index 00000000..062ce741 --- /dev/null +++ b/checks/taskcluster/create_task.py @@ -0,0 +1,78 @@ +""" +A check to verify that tasks can be created. + +Information about the lastest created task is returned. +""" +import logging +import textwrap +from datetime import datetime, timedelta + +import taskcluster +import taskcluster.aio +import taskcluster.exceptions + +from poucave import config +from poucave.typings import CheckResult + +from . import utils as tc_utils + + +logger = logging.getLogger(__name__) + + +TASK_METADATA = { + "owner": config.CONTACT_EMAIL, + "source": config.SOURCE_URL, + "description": textwrap.dedent( + """ + This task is a test and is generated routinely by {config.SERVICE_NAME} + in order to monitor the taskcluster Queue services. It ensures that tasks + are able to be created, and they intentionally have a short expiry + to reduce resource usage. + """ + ), +} + + +async def run( + root_url: str, + queue_id: str = "proj-taskcluster/gw-ci-ubuntu-18-04", + command: str = '/bin/bash -c echo "hola mundo!"', + deadline_seconds: int = 3 * 60 * 60, + expires_seconds: int = 24 * 60 * 60, + max_run_time: int = 10 * 60, + client_id: str = "", + access_token: str = "", + certificate: str = "", +) -> CheckResult: + # Build connection infos from parameters. + options = tc_utils.options_from_params( + root_url, client_id, access_token, certificate + ) + queue = taskcluster.aio.Queue(options) + + name = "task-test" + task_id = taskcluster.stableSlugId()(name) + + now = datetime.utcnow() + deadline = now + timedelta(seconds=deadline_seconds) + expires = now + timedelta(seconds=expires_seconds) + + payload = { + "taskQueueId": queue_id, + "created": now.isoformat(), + "deadline": deadline.isoformat(), + "expires": expires.isoformat(), + "payload": { + "command": [cmd.split() for cmd in command.splitlines()], + "maxRunTime": max_run_time, + }, + "metadata": { + "name": name, + **TASK_METADATA, + }, + } + + status = await queue.createTask(task_id, payload) + + return True, status["status"] diff --git a/poucave/config.py b/poucave/config.py index 7111fb99..b746a46c 100644 --- a/poucave/config.py +++ b/poucave/config.py @@ -33,6 +33,7 @@ REQUESTS_MAX_RETRIES = config("REQUESTS_MAX_RETRIES", default=2, cast=int) REQUESTS_MAX_PARALLEL = config("REQUESTS_MAX_PARALLEL", default=16, cast=int) SENTRY_DSN = config("SENTRY_DSN", default="") +SOURCE_URL = config("SOURCE_URL", default="https://github.com/mozilla-services/poucave") TROUBLESHOOTING_LINK_TEMPLATE = config( "TROUBLESHOOTING_LINK_TEMPLATE", default=( diff --git a/tests/checks/taskcluster/test_create_task.py b/tests/checks/taskcluster/test_create_task.py new file mode 100644 index 00000000..2bdb66ce --- /dev/null +++ b/tests/checks/taskcluster/test_create_task.py @@ -0,0 +1,42 @@ +from datetime import timedelta +from unittest import mock + +import pytest +import taskcluster.exceptions + +from checks.taskcluster.create_task import run +from poucave.utils import utcnow + + +MODULE = "checks.taskcluster.create_task" + +PARAMS = { + "root_url": "http://server", +} + + +async def test_positive(): + class FakeQueue: + async def createTask(self, *args, **kwargs): + return {"status": {"taskId": 42}} + + fake_queue = FakeQueue() + + with mock.patch(f"{MODULE}.taskcluster.aio.Queue", return_value=fake_queue): + status, data = await run(**PARAMS) + + assert status is True + assert data == {"taskId": 42} + + +async def test_negative(): + class FakeQueue: + async def createTask(self, *args, **kwargs): + e = taskcluster.exceptions.TaskclusterRestFailure("", None) + raise e + + fake_queue = FakeQueue() + + with mock.patch(f"{MODULE}.taskcluster.aio.Queue", return_value=fake_queue): + with pytest.raises(taskcluster.exceptions.TaskclusterRestFailure): + await run(**PARAMS) From 8203decfd2b22b9ff110beef68e695f645a674de Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 1 Sep 2021 20:25:42 +0200 Subject: [PATCH 2/5] Lint --- tests/checks/taskcluster/test_create_task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/checks/taskcluster/test_create_task.py b/tests/checks/taskcluster/test_create_task.py index 2bdb66ce..32ba05c7 100644 --- a/tests/checks/taskcluster/test_create_task.py +++ b/tests/checks/taskcluster/test_create_task.py @@ -1,11 +1,9 @@ -from datetime import timedelta from unittest import mock import pytest import taskcluster.exceptions from checks.taskcluster.create_task import run -from poucave.utils import utcnow MODULE = "checks.taskcluster.create_task" From 413a4660f5fef94ec0b9a4b75850f779469693d6 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Thu, 2 Sep 2021 10:29:27 +0200 Subject: [PATCH 3/5] @jwhitlock feedback --- checks/taskcluster/create_task.py | 13 ++++++++----- tests/checks/taskcluster/test_create_task.py | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/checks/taskcluster/create_task.py b/checks/taskcluster/create_task.py index 062ce741..d4e6f03e 100644 --- a/checks/taskcluster/create_task.py +++ b/checks/taskcluster/create_task.py @@ -4,6 +4,7 @@ Information about the lastest created task is returned. """ import logging +import shlex import textwrap from datetime import datetime, timedelta @@ -26,7 +27,7 @@ "description": textwrap.dedent( """ This task is a test and is generated routinely by {config.SERVICE_NAME} - in order to monitor the taskcluster Queue services. It ensures that tasks + in order to monitor the Taskcluster Queue services. It ensures that tasks are able to be created, and they intentionally have a short expiry to reduce resource usage. """ @@ -37,7 +38,8 @@ async def run( root_url: str, queue_id: str = "proj-taskcluster/gw-ci-ubuntu-18-04", - command: str = '/bin/bash -c echo "hola mundo!"', + command: str = """/bin/bash -c 'echo "hola mundo!"'""", + task_source_url: str = "", deadline_seconds: int = 3 * 60 * 60, expires_seconds: int = 24 * 60 * 60, max_run_time: int = 10 * 60, @@ -52,7 +54,7 @@ async def run( queue = taskcluster.aio.Queue(options) name = "task-test" - task_id = taskcluster.stableSlugId()(name) + task_id = taskcluster.stableSlugId()(name) # type: ignore now = datetime.utcnow() deadline = now + timedelta(seconds=deadline_seconds) @@ -64,12 +66,13 @@ async def run( "deadline": deadline.isoformat(), "expires": expires.isoformat(), "payload": { - "command": [cmd.split() for cmd in command.splitlines()], + "command": [shlex.split(cmd) for cmd in command.splitlines()], "maxRunTime": max_run_time, }, "metadata": { - "name": name, **TASK_METADATA, + "name": name, + "source_url": task_source_url or config.SOURCE_URL, }, } diff --git a/tests/checks/taskcluster/test_create_task.py b/tests/checks/taskcluster/test_create_task.py index 32ba05c7..e926a7a0 100644 --- a/tests/checks/taskcluster/test_create_task.py +++ b/tests/checks/taskcluster/test_create_task.py @@ -16,6 +16,7 @@ async def test_positive(): class FakeQueue: async def createTask(self, *args, **kwargs): + self.called_with = args, kwargs return {"status": {"taskId": 42}} fake_queue = FakeQueue() @@ -25,6 +26,10 @@ async def createTask(self, *args, **kwargs): assert status is True assert data == {"taskId": 42} + _, definition = fake_queue.called_with[0] + assert definition["payload"]["command"] == [ + ["/bin/bash", "-c", 'echo "hola mundo!"'] + ] async def test_negative(): From 0cfa475ef6591b3e12fcf140ff66c378e81e90db Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Thu, 2 Sep 2021 10:50:06 +0200 Subject: [PATCH 4/5] Fix source in payload --- checks/taskcluster/create_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/taskcluster/create_task.py b/checks/taskcluster/create_task.py index d4e6f03e..c75fa874 100644 --- a/checks/taskcluster/create_task.py +++ b/checks/taskcluster/create_task.py @@ -72,7 +72,7 @@ async def run( "metadata": { **TASK_METADATA, "name": name, - "source_url": task_source_url or config.SOURCE_URL, + "source": task_source_url or config.SOURCE_URL, }, } From 1c2b00578d5f5534b930fad9f5b32fbc0b2d7eb2 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Thu, 2 Sep 2021 10:50:15 +0200 Subject: [PATCH 5/5] Pete's feedback --- checks/taskcluster/create_task.py | 2 +- tests/checks/taskcluster/test_create_task.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/checks/taskcluster/create_task.py b/checks/taskcluster/create_task.py index c75fa874..61ab7201 100644 --- a/checks/taskcluster/create_task.py +++ b/checks/taskcluster/create_task.py @@ -38,7 +38,7 @@ async def run( root_url: str, queue_id: str = "proj-taskcluster/gw-ci-ubuntu-18-04", - command: str = """/bin/bash -c 'echo "hola mundo!"'""", + command: str = "/bin/echo 'hola mundo!'", task_source_url: str = "", deadline_seconds: int = 3 * 60 * 60, expires_seconds: int = 24 * 60 * 60, diff --git a/tests/checks/taskcluster/test_create_task.py b/tests/checks/taskcluster/test_create_task.py index e926a7a0..894ac9a6 100644 --- a/tests/checks/taskcluster/test_create_task.py +++ b/tests/checks/taskcluster/test_create_task.py @@ -27,9 +27,7 @@ async def createTask(self, *args, **kwargs): assert status is True assert data == {"taskId": 42} _, definition = fake_queue.called_with[0] - assert definition["payload"]["command"] == [ - ["/bin/bash", "-c", 'echo "hola mundo!"'] - ] + assert definition["payload"]["command"] == [["/bin/echo", "hola mundo!"]] async def test_negative():