From ad988b1fc3b5133253ac25bfdb8b8703f9bb6d94 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Mon, 11 Dec 2023 17:18:59 +0100 Subject: [PATCH 01/33] add pre commit hooks --- .pre-commit-config.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..128773f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +ci: + autofix_prs: true +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: debug-statements + - repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + - repo: https://github.com/hadialqattan/pycln + rev: v2.4.0 + hooks: + - id: pycln + args: [--config=pyproject.toml] \ No newline at end of file From 867b98aea57c70a3bcc0bde003ca929b71ea2b58 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Mon, 11 Dec 2023 17:59:06 +0100 Subject: [PATCH 02/33] add dev pytest and coverage ti dev dependencies --- poetry.lock | 141 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 5 ++ 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index d6127d8..6654c3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "certifi" @@ -128,6 +128,17 @@ numpy = "*" constrained-solution-tracking = ["moarchiving"] plotting = ["matplotlib"] +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + [[package]] name = "contourpy" version = "1.1.1" @@ -199,6 +210,70 @@ mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.4.1)", "types-Pill test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] test-no-images = ["pytest", "pytest-cov", "wurlitzer"] +[[package]] +name = "coverage" +version = "7.3.2" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf"}, + {file = "coverage-7.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda"}, + {file = "coverage-7.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a"}, + {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c"}, + {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f"}, + {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6"}, + {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148"}, + {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9"}, + {file = "coverage-7.3.2-cp310-cp310-win32.whl", hash = "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f"}, + {file = "coverage-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611"}, + {file = "coverage-7.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c"}, + {file = "coverage-7.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074"}, + {file = "coverage-7.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a"}, + {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3"}, + {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a"}, + {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1"}, + {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c"}, + {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312"}, + {file = "coverage-7.3.2-cp311-cp311-win32.whl", hash = "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640"}, + {file = "coverage-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2"}, + {file = "coverage-7.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836"}, + {file = "coverage-7.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63"}, + {file = "coverage-7.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216"}, + {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4"}, + {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf"}, + {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf"}, + {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84"}, + {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a"}, + {file = "coverage-7.3.2-cp312-cp312-win32.whl", hash = "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb"}, + {file = "coverage-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed"}, + {file = "coverage-7.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738"}, + {file = "coverage-7.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2"}, + {file = "coverage-7.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2"}, + {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c"}, + {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9"}, + {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82"}, + {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901"}, + {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76"}, + {file = "coverage-7.3.2-cp38-cp38-win32.whl", hash = "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92"}, + {file = "coverage-7.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a"}, + {file = "coverage-7.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce"}, + {file = "coverage-7.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9"}, + {file = "coverage-7.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f"}, + {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25"}, + {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9"}, + {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6"}, + {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc"}, + {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083"}, + {file = "coverage-7.3.2-cp39-cp39-win32.whl", hash = "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce"}, + {file = "coverage-7.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f"}, + {file = "coverage-7.3.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637"}, + {file = "coverage-7.3.2.tar.gz", hash = "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef"}, +] + +[package.extras] +toml = ["tomli"] + [[package]] name = "cycler" version = "0.12.1" @@ -214,6 +289,20 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fonttools" version = "4.44.0" @@ -308,6 +397,17 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "joblib" version = "1.3.2" @@ -633,6 +733,21 @@ files = [ docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "psutil" version = "5.9.6" @@ -675,6 +790,28 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -902,4 +1039,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "52b51c198504f144d7775cc4d2e91f24bc90b4be56adcd9538704458c76c8a9f" +content-hash = "1a4a705b2fb79214bb179e9d08d8e6873bf7dcb990aa18a842c7656ebb958b43" diff --git a/pyproject.toml b/pyproject.toml index d49e528..767b10c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,8 @@ packages = [{ include = "qibo_tii_provider", from = "src" }] python = ">=3.8,<3.12" qibo = ">=0.2.2" requests = "^2.31.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.3" +coverage = "^7.3.2" + From 1698ad48a3915b1a5fc403ef11ff4b930313aa4d Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Tue, 12 Dec 2023 18:53:59 +0100 Subject: [PATCH 03/33] client initialization unit tests --- src/qibo_tii_provider/config.py | 8 ++ src/qibo_tii_provider/tiiprovider.py | 107 ++++++++++++++++++--------- tests/test_tiiqprovider.py | 101 +++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 36 deletions(-) create mode 100644 src/qibo_tii_provider/config.py create mode 100644 tests/test_tiiqprovider.py diff --git a/src/qibo_tii_provider/config.py b/src/qibo_tii_provider/config.py new file mode 100644 index 0000000..6a7a239 --- /dev/null +++ b/src/qibo_tii_provider/config.py @@ -0,0 +1,8 @@ +class MalformedResponseError(Exception): + """Exception raised when server responsed body does not contain expected keys""" + + def __init__( + self, message="Server response body does not contain all the expected keys" + ): + self.message = message + super().__init__(self.message) diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index db2230d..63c008a 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -3,18 +3,20 @@ import tarfile import tempfile import time -from typing import Iterable, Optional +from typing import Dict, Iterable, List, Optional import os import numpy as np import qibo import requests +from .config import MalformedResponseError -QRCCLUSTER_IP=os.environ.get("QRCCLUSTER_IP", "login.qrccluster.com") -QRCCLUSTER_PORT=os.environ.get("QRCCLUSTER_PORT", "8010") -RESULTS_BASE_FOLDER=os.environ.get("RESULTS_BASE_FOLDER", "/tmp/qibo_tii_provider") -SECONDS_BETWEEN_CHECKS=os.environ.get("SECONDS_BETWEEN_CHECKS", 2) + +QRCCLUSTER_IP = os.environ.get("QRCCLUSTER_IP", "login.qrccluster.com") +QRCCLUSTER_PORT = os.environ.get("QRCCLUSTER_PORT", "8010") +RESULTS_BASE_FOLDER = os.environ.get("RESULTS_BASE_FOLDER", "/tmp/qibo_tii_provider") +SECONDS_BETWEEN_CHECKS = os.environ.get("SECONDS_BETWEEN_CHECKS", 2) BASE_URL = f"http://{QRCCLUSTER_IP}:{QRCCLUSTER_PORT}/" @@ -23,7 +25,19 @@ SECONDS_BETWEEN_CHECKS = SECONDS_BETWEEN_CHECKS +# configure logger +import logging + +logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(message)s") +logger = logging.getLogger(__name__) +logging_level = logging.INFO +logger.setLevel(logging_level) + +# TODO: check here that the stream of data +# TODO: split this into two function and unittest the two functionalities: +# - create an archive from a stream iterable +# - extract everything into results_folder def _write_stream_response_to_folder(stream: Iterable, results_folder: Path): """Save the stream to a given folder. @@ -49,6 +63,23 @@ def _write_stream_response_to_folder(stream: Iterable, results_folder: Path): os.remove(archive_path) +def check_response_has_keys(response: requests.models.Response, keys: List[str]): + """Check that the response body contains certain keys. + + + :raises MalformedResponseError: if the server response does not contain all + the expected keys + """ + response_keys = set(response.json().keys()) + expected_keys = set(keys) + missing_keys = expected_keys.difference(response_keys) + + if len(missing_keys): + raise MalformedResponseError( + f"The server response is missing the following keys: {' '.join(missing_keys)}" + ) + + class TIIProvider: """Class to manage the interaction with the QRC cluster.""" @@ -68,10 +99,9 @@ def check_client_server_qibo_versions(self): """ url = BASE_URL + "qibo_version/" response = requests.get(url) - assert ( - response.status_code == 200 - ), f"Failed to send the request to the server, response {response.status_code}" - qibo_server_version = json.loads(response.content)["qibo_version"] + response.raise_for_status() + check_response_has_keys(response, ["qibo_version"]) + qibo_server_version = response.json()["qibo_version"] qibo_local_version = qibo.__version__ assert ( @@ -109,47 +139,49 @@ def run_circuit( :rtype: np.ndarray """ # post circuit to server - print("Post new circuit on the server") - self.__post_circuit(circuit, nshots, device) + logger.info("Post new circuit on the server") + try: + self._post_circuit(circuit, nshots, device) + except: + logger.error() + return # retrieve results - print(f"Job posted on server with pid {self.pid}") - print(f"Check results every {SECONDS_BETWEEN_CHECKS} seconds ...") - result = self.__get_result() + logger.info("Job posted on server with pid %s", self.pid) + logger.info("Check results every %d seconds ...", SECONDS_BETWEEN_CHECKS) + result = self._get_result() return result - def __post_circuit( + def _post_circuit( self, circuit: qibo.Circuit, nshots: int = 100, device: str = "sim" - ): + ) -> int: + """ + + :return: the post request status code. If the post block raises an + exception, the exit code is set to -1. + :rtype: int + """ + # HTTP request + url = BASE_URL + "run_circuit/" payload = { "token": self.token, "circuit": circuit.raw, "nshots": nshots, "device": device, } - url = BASE_URL + "run_circuit/" - - # post circuit - try: - # Send an HTTP request to the server - response = requests.post(url, json=payload) - - # the response should contain the PID to be checked (in the db, store - # an hashed version of the pid, not the actual value) + response = requests.post(url, json=payload) - # Check the response - if response.status_code == 200: - response_content = json.loads(response.content) - self.pid = response_content["pid"] - return response_content["message"] - else: - return "Error. Failed to send the request to the server" + # checks + response.raise_for_status() + check_response_has_keys(response, ["pid", "message"]) - except Exception as e: - return f"Error. An error occurred: {str(e)}" + # save the response + response_content = response.json() + self.pid = response_content["pid"] + return response_content["message"] - def __get_result(self) -> Optional[np.ndarray]: + def _get_result(self) -> Optional[np.ndarray]: """Send requests to server checking whether the job is completed. This function populates the `TIIProvider.result_folder` and @@ -177,7 +209,10 @@ def __get_result(self) -> Optional[np.ndarray]: ) if response.headers["Job-Status"].lower() == "error": - print(f"Job exited with error, check logs in {self.result_folder.as_posix()} folder") + logger.info( + "Job exited with error, check logs in %s folder", + self.result_folder.as_posix(), + ) return None self.result_path = self.result_folder / "results.npy" diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py new file mode 100644 index 0000000..1ed2487 --- /dev/null +++ b/tests/test_tiiqprovider.py @@ -0,0 +1,101 @@ +from typing import Dict, Optional +from unittest.mock import patch, Mock + +from requests.exceptions import HTTPError + +import pytest + +from qibo_tii_provider import tiiprovider +from qibo_tii_provider.config import MalformedResponseError + +PKG = "qibo_tii_provider.tiiprovider" +LOCAL_URL = "http://localhost:8000/" +FAKE_QIBO_VERSION = "0.0.1" + + +@pytest.fixture(autouse=True) +def mock_qrccluster_ip(): + """Ensure that all the requests are made on localhost""" + with patch(f"{PKG}.BASE_URL", LOCAL_URL) as _fixture: + yield _fixture + + +@pytest.fixture(autouse=True) +def mock_local_qibo_version(): + """Ensure that all the requests are made on localhost""" + with patch(f"{PKG}.qibo.__version__", FAKE_QIBO_VERSION) as _fixture: + yield _fixture + + +@pytest.fixture +def mock_request(): + """Returns a mocked get request""" + with patch(f"{PKG}.requests") as _mock_request: + yield _mock_request + + +class MockedResponse: + def __init__(self, status_code: int, json: Optional[Dict] = None): + self.status_code = status_code + self._json = json + + def json(self): + return self._json + + def raise_for_status(self): + if not (200 <= self.status_code < 300): + raise HTTPError + + +def test_check_response_has_keys(): + """Check response body contains the keys""" + keys = ["key1", "key2"] + json = {"key1": 0, "key2": 1} + status_code = 200 + mock_response = MockedResponse(status_code, json) + tiiprovider.check_response_has_keys(mock_response, keys) + + +def test_check_response_has_missing_keys(): + """Check response body contains the keys""" + keys = ["key1", "key2"] + json = {"key1": 0} + status_code = 200 + mock_response = MockedResponse(status_code, json) + with pytest.raises(MalformedResponseError): + tiiprovider.check_response_has_keys(mock_response, keys) + + +def _get_tii_client(): + return tiiprovider.TIIProvider("valid_token") + + +def _execute_check_client_server_qibo_versions( + mock_request, local_qibo_version, remote_qibo_version +): + mock_response = MockedResponse( + status_code=200, json={"qibo_version": remote_qibo_version} + ) + mock_request.get.return_value = mock_response + + with patch(f"{PKG}.qibo.__version__", local_qibo_version): + _get_tii_client() + + +def test_check_client_server_qibo_versions_with_version_match(mock_request: Mock): + _execute_check_client_server_qibo_versions( + mock_request, FAKE_QIBO_VERSION, FAKE_QIBO_VERSION + ) + + mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") + + +def test_check_client_server_qibo_versions_with_version_mismatch(mock_request): + remote_qibo_version = "0.2.2" + + with pytest.raises(AssertionError): + _execute_check_client_server_qibo_versions( + mock_request, FAKE_QIBO_VERSION, remote_qibo_version + ) + + mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") From 2c3b6746326835e7f4c492e3114bec85600bbb02 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Tue, 12 Dec 2023 18:54:52 +0100 Subject: [PATCH 04/33] client with invalid token circuit post test --- tests/test_tiiqprovider.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index 1ed2487..ff09b2a 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -99,3 +99,23 @@ def test_check_client_server_qibo_versions_with_version_mismatch(mock_request): ) mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") + + +class MockedCircuit: + def __init__(self): + self.raw = "raw circuit representation" + + +def test___post_circuit_with_invalid_token(mock_request: Mock): + mock_get_response = MockedResponse( + status_code=200, json={"qibo_version": FAKE_QIBO_VERSION} + ) + mock_request.get.return_value = mock_get_response + + # simulate 404 error due to invalid token + mock_post_response = MockedResponse(status_code=404) + mock_request.post.return_value = mock_post_response + + client = _get_tii_client() + with pytest.raises(HTTPError): + client._post_circuit(MockedCircuit()) From e887d1e2d801b94d3e2ff06032f008f2a4a44138 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Wed, 13 Dec 2023 01:23:03 +0100 Subject: [PATCH 05/33] test post circuit --- src/qibo_tii_provider/config.py | 14 ++++++ src/qibo_tii_provider/tiiprovider.py | 72 +++++++++++++++------------- tests/test_tiiqprovider.py | 72 ++++++++++++++++++++++++---- 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/src/qibo_tii_provider/config.py b/src/qibo_tii_provider/config.py index 6a7a239..9562483 100644 --- a/src/qibo_tii_provider/config.py +++ b/src/qibo_tii_provider/config.py @@ -6,3 +6,17 @@ def __init__( ): self.message = message super().__init__(self.message) + + +class JobPostServerError(Exception): + """Exception raised when server fails to post the job to the queue. + + The client should handle such error to aknowledge that job submission was + not successful without crashing. + """ + + def __init__( + self, message="Server failed to post job to queue" + ): + self.message = message + super().__init__(self.message) \ No newline at end of file diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index 63c008a..c7d17bd 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -10,7 +10,7 @@ import qibo import requests -from .config import MalformedResponseError +from .config import MalformedResponseError, JobPostServerError QRCCLUSTER_IP = os.environ.get("QRCCLUSTER_IP", "login.qrccluster.com") @@ -23,8 +23,6 @@ RESULTS_BASE_FOLDER = Path(RESULTS_BASE_FOLDER) RESULTS_BASE_FOLDER.mkdir(exist_ok=True) -SECONDS_BETWEEN_CHECKS = SECONDS_BETWEEN_CHECKS - # configure logger import logging @@ -34,6 +32,22 @@ logger.setLevel(logging_level) +def wait_for_response_to_get_request(url: str) -> requests.models.Response: + """Wait until the server completes the computation and return the response. + + :param url: the endpoint to make the request + :type url: str + + :return: the response of the get request + :rtype: requests.models.Response + """ + response = requests.get(url) + if response.content == b"Job still in progress": + time.sleep(SECONDS_BETWEEN_CHECKS) + wait_for_response_to_get_request(url) + return response + + # TODO: check here that the stream of data # TODO: split this into two function and unittest the two functionalities: # - create an archive from a stream iterable @@ -140,10 +154,11 @@ def run_circuit( """ # post circuit to server logger.info("Post new circuit on the server") + try: self._post_circuit(circuit, nshots, device) - except: - logger.error() + except JobPostServerError as err: + logger.error(err.message) return # retrieve results @@ -155,13 +170,7 @@ def run_circuit( def _post_circuit( self, circuit: qibo.Circuit, nshots: int = 100, device: str = "sim" - ) -> int: - """ - - :return: the post request status code. If the post block raises an - exception, the exit code is set to -1. - :rtype: int - """ + ): # HTTP request url = BASE_URL + "run_circuit/" payload = { @@ -179,7 +188,9 @@ def _post_circuit( # save the response response_content = response.json() self.pid = response_content["pid"] - return response_content["message"] + + if self.pid is None: + raise JobPostServerError(response_content["message"]) def _get_result(self) -> Optional[np.ndarray]: """Send requests to server checking whether the job is completed. @@ -191,29 +202,22 @@ def _get_result(self) -> Optional[np.ndarray]: the job raised an error. :rtype: Optional[np.ndarray] """ - url = BASE_URL + f"get_result/{self.pid}" - while True: - time.sleep(SECONDS_BETWEEN_CHECKS) - response = requests.get(url) + url = BASE_URL + f"get_result/{self.pid}/" + response = wait_for_response_to_get_request(url) - if response.content == b"Job still in progress": - continue + # create the job results folder + self.result_folder = RESULTS_BASE_FOLDER / self.pid + self.result_folder.mkdir(exist_ok=True) - # create the job results folder - self.result_folder = RESULTS_BASE_FOLDER / self.pid - self.result_folder.mkdir(exist_ok=True) + # Save the stream to disk + _write_stream_response_to_folder(response.iter_content(), self.result_folder) - # Save the stream to disk - _write_stream_response_to_folder( - response.iter_content(), self.result_folder + if response.headers["Job-Status"].lower() == "error": + logger.info( + "Job exited with error, check logs in %s folder", + self.result_folder.as_posix(), ) + return None - if response.headers["Job-Status"].lower() == "error": - logger.info( - "Job exited with error, check logs in %s folder", - self.result_folder.as_posix(), - ) - return None - - self.result_path = self.result_folder / "results.npy" - return qibo.result.load_result(self.result_path) + self.result_path = self.result_folder / "results.npy" + return qibo.result.load_result(self.result_path) diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index ff09b2a..41850d5 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -35,11 +35,12 @@ def mock_request(): class MockedResponse: - def __init__(self, status_code: int, json: Optional[Dict] = None): + def __init__(self, status_code: int, json_data: Optional[Dict] = None): self.status_code = status_code - self._json = json + self._json = json_data + self.content = json_data.get("content") if json_data is not None else None - def json(self): + def json(self) -> Dict: return self._json def raise_for_status(self): @@ -50,18 +51,18 @@ def raise_for_status(self): def test_check_response_has_keys(): """Check response body contains the keys""" keys = ["key1", "key2"] - json = {"key1": 0, "key2": 1} + json_data = {"key1": 0, "key2": 1} status_code = 200 - mock_response = MockedResponse(status_code, json) + mock_response = MockedResponse(status_code, json_data) tiiprovider.check_response_has_keys(mock_response, keys) def test_check_response_has_missing_keys(): """Check response body contains the keys""" keys = ["key1", "key2"] - json = {"key1": 0} + json_data = {"key1": 0} status_code = 200 - mock_response = MockedResponse(status_code, json) + mock_response = MockedResponse(status_code, json_data) with pytest.raises(MalformedResponseError): tiiprovider.check_response_has_keys(mock_response, keys) @@ -74,7 +75,7 @@ def _execute_check_client_server_qibo_versions( mock_request, local_qibo_version, remote_qibo_version ): mock_response = MockedResponse( - status_code=200, json={"qibo_version": remote_qibo_version} + status_code=200, json_data={"qibo_version": remote_qibo_version} ) mock_request.get.return_value = mock_response @@ -106,9 +107,9 @@ def __init__(self): self.raw = "raw circuit representation" -def test___post_circuit_with_invalid_token(mock_request: Mock): +def test__post_circuit_with_invalid_token(mock_request: Mock): mock_get_response = MockedResponse( - status_code=200, json={"qibo_version": FAKE_QIBO_VERSION} + status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} ) mock_request.get.return_value = mock_get_response @@ -119,3 +120,54 @@ def test___post_circuit_with_invalid_token(mock_request: Mock): client = _get_tii_client() with pytest.raises(HTTPError): client._post_circuit(MockedCircuit()) + + +def test__post_circuit_not_successful(mock_request: Mock): + mock_get_response = MockedResponse( + status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} + ) + mock_request.get.return_value = mock_get_response + + # simulate 404 error due to invalid token + json_data = {"pid": None, "message": "post job to queue failed"} + mock_post_response = MockedResponse(status_code=200, json_data=json_data) + mock_request.post.return_value = mock_post_response + + client = _get_tii_client() + with pytest.raises(ValueError): + client._post_circuit(MockedCircuit()) + + +def test__run_circuit_with_unsuccessful_post_to_queue(mock_request: Mock): + mock_get_response = MockedResponse( + status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} + ) + mock_request.get.return_value = mock_get_response + + # simulate 404 error due to invalid token + json_data = {"pid": None, "message": "post job to queue failed"} + mock_post_response = MockedResponse(status_code=200, json_data=json_data) + mock_request.post.return_value = mock_post_response + + client = _get_tii_client() + return_value = client.run_circuit(MockedCircuit()) + + assert return_value is None + + +def test_wait_for_response_to_get_request(mock_request: Mock): + failed_attempts = 3 + url = "http://example.url" + + keep_waiting = MockedResponse( + status_code=200, json_data={"content": b"Job still in progress"} + ) + job_done = MockedResponse(status_code=200) + + mock_request.get.side_effect = [keep_waiting] * failed_attempts + [job_done] + + with patch(f"{PKG}.SECONDS_BETWEEN_CHECKS", 1e-4): + tiiprovider.wait_for_response_to_get_request(url) + + assert mock_request.get.call_count == 4 + From 6c558a6c852add40d60b52984cd5ac0c62b3643f Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Wed, 13 Dec 2023 13:33:27 +0100 Subject: [PATCH 06/33] test StreamingHttpResponse related function --- src/qibo_tii_provider/config.py | 8 +- src/qibo_tii_provider/tiiprovider.py | 76 ++++++++---- tests/test_tiiqprovider.py | 172 +++++++++++++++++++++------ tests/utils_test_tiiqprovider.py | 136 +++++++++++++++++++++ 4 files changed, 324 insertions(+), 68 deletions(-) create mode 100644 tests/utils_test_tiiqprovider.py diff --git a/src/qibo_tii_provider/config.py b/src/qibo_tii_provider/config.py index 9562483..890202f 100644 --- a/src/qibo_tii_provider/config.py +++ b/src/qibo_tii_provider/config.py @@ -10,13 +10,11 @@ def __init__( class JobPostServerError(Exception): """Exception raised when server fails to post the job to the queue. - + The client should handle such error to aknowledge that job submission was not successful without crashing. """ - def __init__( - self, message="Server failed to post job to queue" - ): + def __init__(self, message="Server failed to post job to queue"): self.message = message - super().__init__(self.message) \ No newline at end of file + super().__init__(self.message) diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index c7d17bd..7035f6d 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -1,9 +1,8 @@ -import json from pathlib import Path import tarfile import tempfile import time -from typing import Dict, Iterable, List, Optional +from typing import Iterable, List, Optional import os import numpy as np @@ -41,18 +40,40 @@ def wait_for_response_to_get_request(url: str) -> requests.models.Response: :return: the response of the get request :rtype: requests.models.Response """ - response = requests.get(url) - if response.content == b"Job still in progress": - time.sleep(SECONDS_BETWEEN_CHECKS) - wait_for_response_to_get_request(url) - return response - - -# TODO: check here that the stream of data -# TODO: split this into two function and unittest the two functionalities: -# - create an archive from a stream iterable -# - extract everything into results_folder -def _write_stream_response_to_folder(stream: Iterable, results_folder: Path): + while True: + response = requests.get(url) + if response.content == b"Job still in progress": + time.sleep(SECONDS_BETWEEN_CHECKS) + continue + return response + + +def _write_stream_to_tmp_file(stream: Iterable) -> Path: + """Write chunk of bytes to temporary file. + + The tmp_path should be closed manually. + + :param stream: the stream of bytes chunks to be saved on disk + :type stream: Iterable + + :return: the name of the tempo + + """ + + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + for chunk in stream: + if chunk: + tmp_file.write(chunk) + archive_path = tmp_file.name + return Path(archive_path) + + +def _extract_archive_to_folder(source_archive: Path, destination_folder: Path): + with tarfile.open(source_archive, "r:gz") as archive: + archive.extractall(destination_folder) + + +def _save_and_unpack_stream_response_to_folder(stream: Iterable, results_folder: Path): """Save the stream to a given folder. Internally, save the stream to a temporary archive and extract its contents @@ -63,18 +84,12 @@ def _write_stream_response_to_folder(stream: Iterable, results_folder: Path): :param results_folder: the local path to the results folder :type results_folder: Path """ - # save archive to tempfile - with tempfile.NamedTemporaryFile(delete=False) as archive: - for chunk in stream: - if chunk: - archive.write(chunk) - archive_path = archive.name + archive_path = _write_stream_to_tmp_file(stream) - # extract archive content to target directory - with tarfile.open(archive_path, "r") as archive: - archive.extractall(results_folder) + _extract_archive_to_folder(archive_path, results_folder) - os.remove(archive_path) + # clean up temporary file + archive_path.unlink() def check_response_has_keys(response: requests.models.Response, keys: List[str]): @@ -210,7 +225,18 @@ def _get_result(self) -> Optional[np.ndarray]: self.result_folder.mkdir(exist_ok=True) # Save the stream to disk - _write_stream_response_to_folder(response.iter_content(), self.result_folder) + try: + _save_and_unpack_stream_response_to_folder( + response.iter_content(), self.result_folder + ) + except tarfile.ReadError as err: + logger.error("Catched tarfile ReadError: %s", err) + logger.error( + "The received file is not a valid gzip " + "archive, the result might have to be inspected manually. Find " + "the file at `%s`", + self.result_folder.as_posix(), + ) if response.headers["Job-Status"].lower() == "error": logger.info( diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index 41850d5..13016ba 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -1,12 +1,14 @@ -from typing import Dict, Optional +from pathlib import Path +import tarfile from unittest.mock import patch, Mock +import pytest from requests.exceptions import HTTPError -import pytest +import tests.utils_test_tiiqprovider as utils from qibo_tii_provider import tiiprovider -from qibo_tii_provider.config import MalformedResponseError +from qibo_tii_provider.config import MalformedResponseError, JobPostServerError PKG = "qibo_tii_provider.tiiprovider" LOCAL_URL = "http://localhost:8000/" @@ -34,26 +36,12 @@ def mock_request(): yield _mock_request -class MockedResponse: - def __init__(self, status_code: int, json_data: Optional[Dict] = None): - self.status_code = status_code - self._json = json_data - self.content = json_data.get("content") if json_data is not None else None - - def json(self) -> Dict: - return self._json - - def raise_for_status(self): - if not (200 <= self.status_code < 300): - raise HTTPError - - def test_check_response_has_keys(): """Check response body contains the keys""" keys = ["key1", "key2"] json_data = {"key1": 0, "key2": 1} status_code = 200 - mock_response = MockedResponse(status_code, json_data) + mock_response = utils.MockedResponse(status_code, json_data) tiiprovider.check_response_has_keys(mock_response, keys) @@ -62,7 +50,7 @@ def test_check_response_has_missing_keys(): keys = ["key1", "key2"] json_data = {"key1": 0} status_code = 200 - mock_response = MockedResponse(status_code, json_data) + mock_response = utils.MockedResponse(status_code, json_data) with pytest.raises(MalformedResponseError): tiiprovider.check_response_has_keys(mock_response, keys) @@ -74,7 +62,7 @@ def _get_tii_client(): def _execute_check_client_server_qibo_versions( mock_request, local_qibo_version, remote_qibo_version ): - mock_response = MockedResponse( + mock_response = utils.MockedResponse( status_code=200, json_data={"qibo_version": remote_qibo_version} ) mock_request.get.return_value = mock_response @@ -102,55 +90,50 @@ def test_check_client_server_qibo_versions_with_version_mismatch(mock_request): mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") -class MockedCircuit: - def __init__(self): - self.raw = "raw circuit representation" - - def test__post_circuit_with_invalid_token(mock_request: Mock): - mock_get_response = MockedResponse( + mock_get_response = utils.MockedResponse( status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} ) mock_request.get.return_value = mock_get_response # simulate 404 error due to invalid token - mock_post_response = MockedResponse(status_code=404) + mock_post_response = utils.MockedResponse(status_code=404) mock_request.post.return_value = mock_post_response client = _get_tii_client() with pytest.raises(HTTPError): - client._post_circuit(MockedCircuit()) + client._post_circuit(utils.MockedCircuit()) def test__post_circuit_not_successful(mock_request: Mock): - mock_get_response = MockedResponse( + mock_get_response = utils.MockedResponse( status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} ) mock_request.get.return_value = mock_get_response # simulate 404 error due to invalid token json_data = {"pid": None, "message": "post job to queue failed"} - mock_post_response = MockedResponse(status_code=200, json_data=json_data) + mock_post_response = utils.MockedResponse(status_code=200, json_data=json_data) mock_request.post.return_value = mock_post_response client = _get_tii_client() - with pytest.raises(ValueError): - client._post_circuit(MockedCircuit()) + with pytest.raises(JobPostServerError): + client._post_circuit(utils.MockedCircuit()) def test__run_circuit_with_unsuccessful_post_to_queue(mock_request: Mock): - mock_get_response = MockedResponse( + mock_get_response = utils.MockedResponse( status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} ) mock_request.get.return_value = mock_get_response # simulate 404 error due to invalid token json_data = {"pid": None, "message": "post job to queue failed"} - mock_post_response = MockedResponse(status_code=200, json_data=json_data) + mock_post_response = utils.MockedResponse(status_code=200, json_data=json_data) mock_request.post.return_value = mock_post_response client = _get_tii_client() - return_value = client.run_circuit(MockedCircuit()) + return_value = client.run_circuit(utils.MockedCircuit()) assert return_value is None @@ -159,15 +142,128 @@ def test_wait_for_response_to_get_request(mock_request: Mock): failed_attempts = 3 url = "http://example.url" - keep_waiting = MockedResponse( + keep_waiting = utils.MockedResponse( status_code=200, json_data={"content": b"Job still in progress"} ) - job_done = MockedResponse(status_code=200) + job_done = utils.MockedResponse(status_code=200) mock_request.get.side_effect = [keep_waiting] * failed_attempts + [job_done] with patch(f"{PKG}.SECONDS_BETWEEN_CHECKS", 1e-4): tiiprovider.wait_for_response_to_get_request(url) - assert mock_request.get.call_count == 4 + assert mock_request.get.call_count == failed_attempts + 1 + + +# STREAM = ["line1\n", "line2\n"] +ARCHIVE_NAME = "file.tar.gz" + + +@patch(f"{PKG}.tempfile") +def test__write_stream_to_tmp_file_with_simple_text_stream( + mock_tempfile: Mock, tmp_path: Path +): + """ + The test contains the following checks: + + - a new temporary file is created to a specific direction + - the content of the temporary file contains equals the one given + """ + stream = [b"line1\n", b"line2\n"] + file_path = tmp_path / ARCHIVE_NAME + + mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) + + assert not file_path.is_file() + + result_path = tiiprovider._write_stream_to_tmp_file(stream) + + assert result_path == file_path + assert result_path.is_file() + + assert result_path.read_bytes() == b"".join(stream) + + +@patch(f"{PKG}.tempfile") +def test__write_stream_to_tmp_file(mock_tempfile: Mock, tmp_path: Path): + """ + The test contains the following checks: + + - a new temporary file is created to a specific direction + - the content of the temporary file contains equals the one given + """ + file_path = tmp_path / ARCHIVE_NAME + stream, members, members_contents = utils.get_in_memory_fake_archive_stream( + file_path + ) + + mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) + + assert not file_path.is_file() + + result_path = tiiprovider._write_stream_to_tmp_file(stream) + + assert result_path == file_path + assert result_path.is_file() + + # load the archive in memory and check that the members and the contents + # match with the expected ones + with tarfile.open(result_path, "r:gz") as archive: + result_members = sorted(archive.getnames()) + assert result_members == members + for member, member_content in zip(members, members_contents): + with archive.extractfile(member) as result_member: + result_content = result_member.read() + assert result_content == member_content + + +def test__extract_archive_to_folder_with_non_archive_input(tmp_path): + file_path = tmp_path / "file.txt" + file_path.write_text("test content") + + destination_folder = tmp_path / "destination_folder" + destination_folder.mkdir() + + with pytest.raises(tarfile.TarError): + tiiprovider._extract_archive_to_folder(file_path, destination_folder) + + +def test__extract_archive_to_folder(tmp_path): + archive_path = tmp_path / ARCHIVE_NAME + destination_folder = tmp_path / "destination_folder" + destination_folder.mkdir() + + members, members_contents = utils.create_fake_archive(archive_path) + + tiiprovider._extract_archive_to_folder(archive_path, destination_folder) + + result_members = [] + result_members_contents = [] + for member_path in sorted(destination_folder.iterdir()): + result_members.append(member_path.name) + result_members_contents.append(member_path.read_bytes()) + + assert result_members == members + assert result_members_contents == members_contents + + +@patch(f"{PKG}.tempfile") +def test__save_and_unpack_stream_response_to_folder( + mock_tempfile: Mock, tmp_path: Path +): + file_path = tmp_path / ARCHIVE_NAME + destination_folder = tmp_path / "destination_folder" + destination_folder.mkdir() + + mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) + + stream, _, _ = utils.get_in_memory_fake_archive_stream(file_path) + + assert not file_path.is_file() + + tiiprovider._save_and_unpack_stream_response_to_folder(stream, destination_folder) + + # the archive should have been removed + assert not file_path.is_file() + diff --git a/tests/utils_test_tiiqprovider.py b/tests/utils_test_tiiqprovider.py new file mode 100644 index 0000000..d758c3c --- /dev/null +++ b/tests/utils_test_tiiqprovider.py @@ -0,0 +1,136 @@ +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from requests.exceptions import HTTPError + + +class MockedResponse: + """A fake representation of a `requests.response` object""" + + def __init__(self, status_code: int, json_data: Optional[Dict] = None): + self.status_code = status_code + self._json = json_data + self.content = json_data.get("content") if json_data is not None else None + self._iter_content = json_data.get("content") if json_data is not None else None + + def json(self) -> Dict: + return self._json + + def iter_content(self): + return self._iter_content + + def raise_for_status(self): + if 400 <= self.status_code < 500: + http_error_msg = f"{self.status_code} Client Error" + + elif 500 <= self.status_code < 600: + http_error_msg = f"{self.status_code} Server Error" + else: + http_error_msg = "" + + if http_error_msg: + raise HTTPError(http_error_msg) + + +class MockedCircuit: + """A fake representation of a Qibo quantum circuit""" + + def __init__(self): + self.raw = "raw circuit representation" + + +import io +import tarfile + + +class FakeStreamingHttpResponse: + """A fake representation of Django StreamingHttpResponse""" + + def __init__(self, tar_gz_bytes): + self.tar_gz_bytes = tar_gz_bytes + + def __iter__(self): + # Create a tarfile object from the bytes stream + tar_stream = io.BytesIO(self.tar_gz_bytes) + with tarfile.open(fileobj=tar_stream, mode="r:gz") as tar: + for tar_info in tar: + # Yield each byte of the file's content + with tar.extractfile(tar_info) as file: + while byte := file.read(1): + yield byte + + +def _generic_create_archive_(archive_path, get_file_context_manager_fn): + members = ["member1.txt", "member2.txt"] + members_contents = [ + b"This is the content of member1.txt.", + b"This is the content of member2.txt.", + ] + + with get_file_context_manager_fn() as tar: + for member, contents in zip(members, members_contents): + member_info = tarfile.TarInfo(member) + member_info.size = len(contents) + tar.addfile(member_info, io.BytesIO(contents)) + + return members, members_contents + + +def create_fake_archive(archive_path: Path) -> Tuple[List[str], List[str]]: + """Create a .tar.gz archive with fake members and + + + :param archive_path: the destination path for the archive + :type archive_path: Path + + :return: the list with the archive file members + :rtype: List[str] + :return: the list with the contents of each archive file member + :rtype: List[str] + """ + members, members_contents = _generic_create_archive_( + archive_path, lambda: tarfile.open(archive_path, "w:gz") + ) + return members, members_contents + + +def create_in_memory_fake_archive(archive_path: Path): + with io.BytesIO() as buffer: + members, members_contents = _generic_create_archive_( + archive_path, lambda: tarfile.open(fileobj=buffer, mode="w:gz") + ) + archive_as_bytes = buffer.getvalue() + return archive_as_bytes, members, members_contents + + +class TarGzFileStreamer: + def __init__(self, data, chunk_size=128): + self.data = data + self.chunk_size = chunk_size + self.size = len(data) + + def __iter__(self): + for i in range(0, len(self.data), self.chunk_size): + yield self.data[i : i + self.chunk_size] + + +def get_in_memory_fake_archive_stream(archive_path): + archive_as_bytes, members, members_contents = create_in_memory_fake_archive( + archive_path + ) + return TarGzFileStreamer(archive_as_bytes), members, members_contents + + +def get_fake_tmp_file_class(file_path: Path): + class TmpFile: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + self.opened_file = open(file_path, "wb") + return self.opened_file + + def __exit__(self, exc_type, exc_value, exc_tb): + self.opened_file.close() + + return TmpFile From 89dc8b55fc4c146766614d395ce265a63ef5ef18 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 11:27:36 +0100 Subject: [PATCH 07/33] improve coverage to 97% --- src/qibo_tii_provider/tiiprovider.py | 19 ++-- tests/test_tiiqprovider.py | 161 +++++++++++++++++++++------ tests/utils_test_tiiqprovider.py | 13 +-- 3 files changed, 139 insertions(+), 54 deletions(-) diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index 7035f6d..3104ea8 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -59,7 +59,6 @@ def _write_stream_to_tmp_file(stream: Iterable) -> Path: :return: the name of the tempo """ - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: for chunk in stream: if chunk: @@ -210,8 +209,8 @@ def _post_circuit( def _get_result(self) -> Optional[np.ndarray]: """Send requests to server checking whether the job is completed. - This function populates the `TIIProvider.result_folder` and - `TIIProvider.result_path` attributes. + This function populates the `TIIProvider.results_folder` and + `TIIProvider.results_path` attributes. :return: the numpy array with the results of the computation. None if the job raised an error. @@ -221,13 +220,13 @@ def _get_result(self) -> Optional[np.ndarray]: response = wait_for_response_to_get_request(url) # create the job results folder - self.result_folder = RESULTS_BASE_FOLDER / self.pid - self.result_folder.mkdir(exist_ok=True) + self.results_folder = RESULTS_BASE_FOLDER / self.pid + self.results_folder.mkdir(exist_ok=True) # Save the stream to disk try: _save_and_unpack_stream_response_to_folder( - response.iter_content(), self.result_folder + response.iter_content(), self.results_folder ) except tarfile.ReadError as err: logger.error("Catched tarfile ReadError: %s", err) @@ -235,15 +234,15 @@ def _get_result(self) -> Optional[np.ndarray]: "The received file is not a valid gzip " "archive, the result might have to be inspected manually. Find " "the file at `%s`", - self.result_folder.as_posix(), + self.results_folder.as_posix(), ) if response.headers["Job-Status"].lower() == "error": logger.info( "Job exited with error, check logs in %s folder", - self.result_folder.as_posix(), + self.results_folder.as_posix(), ) return None - self.result_path = self.result_folder / "results.npy" - return qibo.result.load_result(self.result_path) + self.results_path = self.results_folder / "results.npy" + return qibo.result.load_result(self.results_path) diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index 13016ba..f261482 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -1,5 +1,6 @@ from pathlib import Path import tarfile +from typing import Callable from unittest.mock import patch, Mock import pytest @@ -13,6 +14,8 @@ PKG = "qibo_tii_provider.tiiprovider" LOCAL_URL = "http://localhost:8000/" FAKE_QIBO_VERSION = "0.0.1" +FAKE_PID = "123" +ARCHIVE_NAME = "file.tar.gz" @pytest.fixture(autouse=True) @@ -23,10 +26,12 @@ def mock_qrccluster_ip(): @pytest.fixture(autouse=True) -def mock_local_qibo_version(): +def mock_qibo(): """Ensure that all the requests are made on localhost""" - with patch(f"{PKG}.qibo.__version__", FAKE_QIBO_VERSION) as _fixture: - yield _fixture + with patch(f"{PKG}.qibo") as _mock_qibo: + _mock_qibo.__version__ = FAKE_QIBO_VERSION + _mock_qibo.result.load_result.side_effect = lambda x: x + yield _mock_qibo @pytest.fixture @@ -36,6 +41,18 @@ def mock_request(): yield _mock_request +@pytest.fixture +def archive_path(tmp_path): + return tmp_path / ARCHIVE_NAME + + +@pytest.fixture +def mock_tempfile(archive_path): + with patch(f"{PKG}.tempfile") as _mock_tempfile: + _mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(archive_path) + yield _mock_tempfile + + def test_check_response_has_keys(): """Check response body contains the keys""" keys = ["key1", "key2"] @@ -66,9 +83,7 @@ def _execute_check_client_server_qibo_versions( status_code=200, json_data={"qibo_version": remote_qibo_version} ) mock_request.get.return_value = mock_response - - with patch(f"{PKG}.qibo.__version__", local_qibo_version): - _get_tii_client() + _get_tii_client() def test_check_client_server_qibo_versions_with_version_match(mock_request: Mock): @@ -155,13 +170,8 @@ def test_wait_for_response_to_get_request(mock_request: Mock): assert mock_request.get.call_count == failed_attempts + 1 -# STREAM = ["line1\n", "line2\n"] -ARCHIVE_NAME = "file.tar.gz" - - -@patch(f"{PKG}.tempfile") def test__write_stream_to_tmp_file_with_simple_text_stream( - mock_tempfile: Mock, tmp_path: Path + mock_tempfile: Mock, archive_path: Path ): """ The test contains the following checks: @@ -170,40 +180,32 @@ def test__write_stream_to_tmp_file_with_simple_text_stream( - the content of the temporary file contains equals the one given """ stream = [b"line1\n", b"line2\n"] - file_path = tmp_path / ARCHIVE_NAME - mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) - - assert not file_path.is_file() + assert not archive_path.is_file() result_path = tiiprovider._write_stream_to_tmp_file(stream) - assert result_path == file_path + assert result_path == archive_path assert result_path.is_file() - assert result_path.read_bytes() == b"".join(stream) -@patch(f"{PKG}.tempfile") -def test__write_stream_to_tmp_file(mock_tempfile: Mock, tmp_path: Path): +def test__write_stream_to_tmp_file(mock_tempfile: Mock, archive_path: Path): """ The test contains the following checks: - a new temporary file is created to a specific direction - the content of the temporary file contains equals the one given """ - file_path = tmp_path / ARCHIVE_NAME stream, members, members_contents = utils.get_in_memory_fake_archive_stream( - file_path + archive_path ) - mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) - - assert not file_path.is_file() + assert not archive_path.is_file() result_path = tiiprovider._write_stream_to_tmp_file(stream) - assert result_path == file_path + assert result_path == archive_path assert result_path.is_file() # load the archive in memory and check that the members and the contents @@ -224,12 +226,11 @@ def test__extract_archive_to_folder_with_non_archive_input(tmp_path): destination_folder = tmp_path / "destination_folder" destination_folder.mkdir() - with pytest.raises(tarfile.TarError): + with pytest.raises(tarfile.ReadError): tiiprovider._extract_archive_to_folder(file_path, destination_folder) -def test__extract_archive_to_folder(tmp_path): - archive_path = tmp_path / ARCHIVE_NAME +def test__extract_archive_to_folder(tmp_path, archive_path): destination_folder = tmp_path / "destination_folder" destination_folder.mkdir() @@ -247,23 +248,109 @@ def test__extract_archive_to_folder(tmp_path): assert result_members_contents == members_contents -@patch(f"{PKG}.tempfile") def test__save_and_unpack_stream_response_to_folder( - mock_tempfile: Mock, tmp_path: Path + mock_tempfile: Mock, archive_path: Path, tmp_path: Path ): - file_path = tmp_path / ARCHIVE_NAME destination_folder = tmp_path / "destination_folder" destination_folder.mkdir() - mock_tempfile.NamedTemporaryFile = utils.get_fake_tmp_file_class(file_path) + stream, _, _ = utils.get_in_memory_fake_archive_stream(archive_path) - stream, _, _ = utils.get_in_memory_fake_archive_stream(file_path) - - assert not file_path.is_file() + assert not archive_path.is_file() tiiprovider._save_and_unpack_stream_response_to_folder(stream, destination_folder) # the archive should have been removed - assert not file_path.is_file() + assert not archive_path.is_file() + + +def _get_request_side_effect(job_status: str = "success") -> Callable: + """Return a callable mock for the get request function + + Job status parameter controls the response header of `get_result/{pid}` + endpoint. + + :param job_status: the Job-Status header of the mocked response + :type job_status: str + + :return: the get request side effect function + :rtype: Callable + """ + + def _request_side_effect(url): + if url == LOCAL_URL + "qibo_version/": + return utils.MockedResponse( + status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} + ) + if url == LOCAL_URL + f"get_result/{FAKE_PID}/": + stream, _, _ = utils.get_in_memory_fake_archive_stream(archive_path) + json_data = { + "content": None, + "iter_content": stream, + "headers": {"Job-Status": job_status}, + } + return utils.MockedResponse(status_code=200, json_data=json_data) + + return _request_side_effect + + +def _post_request_side_effect(url, json): + if url == LOCAL_URL + "run_circuit/": + json_data = {"pid": FAKE_PID, "message": "Success. Job posted"} + return utils.MockedResponse(status_code=200, json_data=json_data) + + +@pytest.fixture +def mock_all_request_methods(): + with patch(f"{PKG}.requests") as _mock_request: + _mock_request.get.side_effect = _get_request_side_effect() + _mock_request.post.side_effect = _post_request_side_effect + yield _mock_request + + +def _generic_test__get_results_fn(results_base_folder: Path): + results_base_folder.mkdir() + + with patch(f"{PKG}.RESULTS_BASE_FOLDER", results_base_folder): + client = _get_tii_client() + client.pid = FAKE_PID + return client._get_result() + + +def test__get_result(mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path): + results_base_folder = tmp_path / "results" + expected_array_path = results_base_folder / FAKE_PID / "results.npy" + + result = _generic_test__get_results_fn(results_base_folder) + + mock_qibo.result.load_result.assert_called_once_with(expected_array_path) + assert result == expected_array_path + + +def test__get_result_with_job_status_error( + mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path +): + mock_all_request_methods.get.side_effect = _get_request_side_effect( + job_status="error" + ) + + results_base_folder = tmp_path / "results" + + result = _generic_test__get_results_fn(results_base_folder) + + mock_qibo.result.load_result.assert_not_called() + assert result is None + + +def test__run_circuit(mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path): + results_base_folder = tmp_path / "results" + expected_array_path = results_base_folder / FAKE_PID / "results.npy" + + results_base_folder.mkdir() + with patch(f"{PKG}.RESULTS_BASE_FOLDER", results_base_folder): + client = _get_tii_client() + client.pid = FAKE_PID + result = client.run_circuit(utils.MockedCircuit()) + assert result == expected_array_path diff --git a/tests/utils_test_tiiqprovider.py b/tests/utils_test_tiiqprovider.py index d758c3c..3922d4d 100644 --- a/tests/utils_test_tiiqprovider.py +++ b/tests/utils_test_tiiqprovider.py @@ -1,4 +1,6 @@ +import io from pathlib import Path +import tarfile from typing import Dict, List, Optional, Tuple from requests.exceptions import HTTPError @@ -9,9 +11,10 @@ class MockedResponse: def __init__(self, status_code: int, json_data: Optional[Dict] = None): self.status_code = status_code - self._json = json_data - self.content = json_data.get("content") if json_data is not None else None - self._iter_content = json_data.get("content") if json_data is not None else None + self._json = json_data or {} + self.headers = json_data.get("headers") + self.content = json_data.get("content") + self._iter_content = json_data.get("iter_content") def json(self) -> Dict: return self._json @@ -39,10 +42,6 @@ def __init__(self): self.raw = "raw circuit representation" -import io -import tarfile - - class FakeStreamingHttpResponse: """A fake representation of Django StreamingHttpResponse""" From d2426c9c7625df3e61b17f825b4ff3402a9e4859 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 13:45:47 +0100 Subject: [PATCH 08/33] complete tiiprovider test coverage --- src/qibo_tii_provider/tiiprovider.py | 1 + tests/test_tiiqprovider.py | 233 ++++++++++++--------------- tests/utils_test_tiiqprovider.py | 38 ++--- 3 files changed, 122 insertions(+), 150 deletions(-) diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index 3104ea8..09c9dd7 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -236,6 +236,7 @@ def _get_result(self) -> Optional[np.ndarray]: "the file at `%s`", self.results_folder.as_posix(), ) + return if response.headers["Job-Status"].lower() == "error": logger.info( diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index f261482..fbd787d 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -1,7 +1,7 @@ from pathlib import Path import tarfile from typing import Callable -from unittest.mock import patch, Mock +from unittest.mock import call, patch, Mock import pytest from requests.exceptions import HTTPError @@ -34,10 +34,55 @@ def mock_qibo(): yield _mock_qibo +@pytest.fixture +def results_base_folder(tmp_path: Path): + results_base_folder = tmp_path / "results" + results_base_folder.mkdir() + with patch(f"{PKG}.RESULTS_BASE_FOLDER", results_base_folder): + yield results_base_folder + + +def _get_request_side_effect(job_status: str = "success") -> Callable: + """Return a callable mock for the get request function + + Job status parameter controls the response header of `get_result/{pid}` + endpoint. + + :param job_status: the Job-Status header of the mocked response + :type job_status: str + + :return: the get request side effect function + :rtype: Callable + """ + + def _request_side_effect(url): + if url == LOCAL_URL + "qibo_version/": + return utils.MockedResponse( + status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} + ) + if url == LOCAL_URL + f"get_result/{FAKE_PID}/": + stream, _, _ = utils.get_in_memory_fake_archive_stream() + json_data = { + "content": None, + "iter_content": stream, + "headers": {"Job-Status": job_status}, + } + return utils.MockedResponse(status_code=200, json_data=json_data) + + return _request_side_effect + + +def _post_request_side_effect(url, json): + if url == LOCAL_URL + "run_circuit/": + json_data = {"pid": FAKE_PID, "message": "Success. Job posted"} + return utils.MockedResponse(status_code=200, json_data=json_data) + + @pytest.fixture def mock_request(): - """Returns a mocked get request""" with patch(f"{PKG}.requests") as _mock_request: + _mock_request.get.side_effect = _get_request_side_effect() + _mock_request.post.side_effect = _post_request_side_effect yield _mock_request @@ -76,44 +121,32 @@ def _get_tii_client(): return tiiprovider.TIIProvider("valid_token") -def _execute_check_client_server_qibo_versions( - mock_request, local_qibo_version, remote_qibo_version -): - mock_response = utils.MockedResponse( - status_code=200, json_data={"qibo_version": remote_qibo_version} - ) - mock_request.get.return_value = mock_response - _get_tii_client() - - def test_check_client_server_qibo_versions_with_version_match(mock_request: Mock): - _execute_check_client_server_qibo_versions( - mock_request, FAKE_QIBO_VERSION, FAKE_QIBO_VERSION - ) - + _get_tii_client() mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") -def test_check_client_server_qibo_versions_with_version_mismatch(mock_request): +def test_check_client_server_qibo_versions_with_version_mismatch(mock_request: Mock): remote_qibo_version = "0.2.2" - with pytest.raises(AssertionError): - _execute_check_client_server_qibo_versions( - mock_request, FAKE_QIBO_VERSION, remote_qibo_version + def _new_side_effect(url): + return utils.MockedResponse( + status_code=200, json_data={"qibo_version": remote_qibo_version} ) + mock_request.get.side_effect = _new_side_effect + + with pytest.raises(AssertionError): + _get_tii_client() + mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") def test__post_circuit_with_invalid_token(mock_request: Mock): - mock_get_response = utils.MockedResponse( - status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} - ) - mock_request.get.return_value = mock_get_response + def _new_side_effect(url, json): + return utils.MockedResponse(status_code=404) - # simulate 404 error due to invalid token - mock_post_response = utils.MockedResponse(status_code=404) - mock_request.post.return_value = mock_post_response + mock_request.post.side_effect = _new_side_effect client = _get_tii_client() with pytest.raises(HTTPError): @@ -121,15 +154,11 @@ def test__post_circuit_with_invalid_token(mock_request: Mock): def test__post_circuit_not_successful(mock_request: Mock): - mock_get_response = utils.MockedResponse( - status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} - ) - mock_request.get.return_value = mock_get_response + def _new_side_effect(url, json): + json_data = {"pid": None, "message": "post job to queue failed"} + return utils.MockedResponse(status_code=200, json_data=json_data) - # simulate 404 error due to invalid token - json_data = {"pid": None, "message": "post job to queue failed"} - mock_post_response = utils.MockedResponse(status_code=200, json_data=json_data) - mock_request.post.return_value = mock_post_response + mock_request.post.side_effect = _new_side_effect client = _get_tii_client() with pytest.raises(JobPostServerError): @@ -137,15 +166,11 @@ def test__post_circuit_not_successful(mock_request: Mock): def test__run_circuit_with_unsuccessful_post_to_queue(mock_request: Mock): - mock_get_response = utils.MockedResponse( - status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} - ) - mock_request.get.return_value = mock_get_response + def _new_side_effect(url, json): + json_data = {"pid": None, "message": "post job to queue failed"} + return utils.MockedResponse(status_code=200, json_data=json_data) - # simulate 404 error due to invalid token - json_data = {"pid": None, "message": "post job to queue failed"} - mock_post_response = utils.MockedResponse(status_code=200, json_data=json_data) - mock_request.post.return_value = mock_post_response + mock_request.post.side_effect = _new_side_effect client = _get_tii_client() return_value = client.run_circuit(utils.MockedCircuit()) @@ -197,9 +222,7 @@ def test__write_stream_to_tmp_file(mock_tempfile: Mock, archive_path: Path): - a new temporary file is created to a specific direction - the content of the temporary file contains equals the one given """ - stream, members, members_contents = utils.get_in_memory_fake_archive_stream( - archive_path - ) + stream, members, members_contents = utils.get_in_memory_fake_archive_stream() assert not archive_path.is_file() @@ -223,24 +246,31 @@ def test__extract_archive_to_folder_with_non_archive_input(tmp_path): file_path = tmp_path / "file.txt" file_path.write_text("test content") - destination_folder = tmp_path / "destination_folder" - destination_folder.mkdir() - with pytest.raises(tarfile.ReadError): - tiiprovider._extract_archive_to_folder(file_path, destination_folder) + tiiprovider._extract_archive_to_folder(file_path, tmp_path) -def test__extract_archive_to_folder(tmp_path, archive_path): - destination_folder = tmp_path / "destination_folder" - destination_folder.mkdir() +@patch( + f"{PKG}._save_and_unpack_stream_response_to_folder", utils.raise_tarfile_readerror +) +def test__get_result_handles_tarfile_readerror(mock_request, results_base_folder): + file_path = results_base_folder / "file.txt" + file_path.write_text("test content") + client = _get_tii_client() + result = client.run_circuit(utils.MockedCircuit()) + + assert result is None + + +def test__extract_archive_to_folder(archive_path, results_base_folder): members, members_contents = utils.create_fake_archive(archive_path) - tiiprovider._extract_archive_to_folder(archive_path, destination_folder) + tiiprovider._extract_archive_to_folder(archive_path, results_base_folder) result_members = [] result_members_contents = [] - for member_path in sorted(destination_folder.iterdir()): + for member_path in sorted(results_base_folder.iterdir()): result_members.append(member_path.name) result_members_contents.append(member_path.read_bytes()) @@ -249,108 +279,47 @@ def test__extract_archive_to_folder(tmp_path, archive_path): def test__save_and_unpack_stream_response_to_folder( - mock_tempfile: Mock, archive_path: Path, tmp_path: Path + mock_tempfile: Mock, archive_path: Path, results_base_folder: Path ): - destination_folder = tmp_path / "destination_folder" - destination_folder.mkdir() - - stream, _, _ = utils.get_in_memory_fake_archive_stream(archive_path) + stream, _, _ = utils.get_in_memory_fake_archive_stream() assert not archive_path.is_file() - tiiprovider._save_and_unpack_stream_response_to_folder(stream, destination_folder) + tiiprovider._save_and_unpack_stream_response_to_folder(stream, results_base_folder) # the archive should have been removed assert not archive_path.is_file() -def _get_request_side_effect(job_status: str = "success") -> Callable: - """Return a callable mock for the get request function - - Job status parameter controls the response header of `get_result/{pid}` - endpoint. - - :param job_status: the Job-Status header of the mocked response - :type job_status: str - - :return: the get request side effect function - :rtype: Callable - """ - - def _request_side_effect(url): - if url == LOCAL_URL + "qibo_version/": - return utils.MockedResponse( - status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} - ) - if url == LOCAL_URL + f"get_result/{FAKE_PID}/": - stream, _, _ = utils.get_in_memory_fake_archive_stream(archive_path) - json_data = { - "content": None, - "iter_content": stream, - "headers": {"Job-Status": job_status}, - } - return utils.MockedResponse(status_code=200, json_data=json_data) - - return _request_side_effect - - -def _post_request_side_effect(url, json): - if url == LOCAL_URL + "run_circuit/": - json_data = {"pid": FAKE_PID, "message": "Success. Job posted"} - return utils.MockedResponse(status_code=200, json_data=json_data) - - -@pytest.fixture -def mock_all_request_methods(): - with patch(f"{PKG}.requests") as _mock_request: - _mock_request.get.side_effect = _get_request_side_effect() - _mock_request.post.side_effect = _post_request_side_effect - yield _mock_request - - -def _generic_test__get_results_fn(results_base_folder: Path): - results_base_folder.mkdir() - - with patch(f"{PKG}.RESULTS_BASE_FOLDER", results_base_folder): - client = _get_tii_client() - client.pid = FAKE_PID - return client._get_result() - - -def test__get_result(mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path): - results_base_folder = tmp_path / "results" +def test__get_result(mock_qibo, mock_request, mock_tempfile, results_base_folder): expected_array_path = results_base_folder / FAKE_PID / "results.npy" - result = _generic_test__get_results_fn(results_base_folder) + client = _get_tii_client() + client.pid = FAKE_PID + result = client._get_result() mock_qibo.result.load_result.assert_called_once_with(expected_array_path) assert result == expected_array_path def test__get_result_with_job_status_error( - mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path + mock_qibo, mock_request, mock_tempfile, results_base_folder ): - mock_all_request_methods.get.side_effect = _get_request_side_effect( - job_status="error" - ) + mock_request.get.side_effect = _get_request_side_effect(job_status="error") - results_base_folder = tmp_path / "results" - - result = _generic_test__get_results_fn(results_base_folder) + client = _get_tii_client() + client.pid = FAKE_PID + result = client._get_result() mock_qibo.result.load_result.assert_not_called() assert result is None -def test__run_circuit(mock_qibo, mock_all_request_methods, mock_tempfile, tmp_path): - results_base_folder = tmp_path / "results" +def test__run_circuit(mock_qibo, mock_request, mock_tempfile, results_base_folder): expected_array_path = results_base_folder / FAKE_PID / "results.npy" - results_base_folder.mkdir() - - with patch(f"{PKG}.RESULTS_BASE_FOLDER", results_base_folder): - client = _get_tii_client() - client.pid = FAKE_PID - result = client.run_circuit(utils.MockedCircuit()) + client = _get_tii_client() + client.pid = FAKE_PID + result = client.run_circuit(utils.MockedCircuit()) assert result == expected_array_path diff --git a/tests/utils_test_tiiqprovider.py b/tests/utils_test_tiiqprovider.py index 3922d4d..d0c94e1 100644 --- a/tests/utils_test_tiiqprovider.py +++ b/tests/utils_test_tiiqprovider.py @@ -1,7 +1,7 @@ import io from pathlib import Path import tarfile -from typing import Dict, List, Optional, Tuple +from typing import Dict, Generator, List, Optional, Tuple from requests.exceptions import HTTPError @@ -12,9 +12,9 @@ class MockedResponse: def __init__(self, status_code: int, json_data: Optional[Dict] = None): self.status_code = status_code self._json = json_data or {} - self.headers = json_data.get("headers") - self.content = json_data.get("content") - self._iter_content = json_data.get("iter_content") + self.headers = self._json.get("headers") + self.content = self._json.get("content") + self._iter_content = self._json.get("iter_content") def json(self) -> Dict: return self._json @@ -59,7 +59,7 @@ def __iter__(self): yield byte -def _generic_create_archive_(archive_path, get_file_context_manager_fn): +def _generic_create_archive_(get_file_context_manager_fn): members = ["member1.txt", "member2.txt"] members_contents = [ b"This is the content of member1.txt.", @@ -75,7 +75,7 @@ def _generic_create_archive_(archive_path, get_file_context_manager_fn): return members, members_contents -def create_fake_archive(archive_path: Path) -> Tuple[List[str], List[str]]: +def create_fake_archive(archive_path: Path) -> Tuple[List[str], List[bytes]]: """Create a .tar.gz archive with fake members and @@ -85,39 +85,37 @@ def create_fake_archive(archive_path: Path) -> Tuple[List[str], List[str]]: :return: the list with the archive file members :rtype: List[str] :return: the list with the contents of each archive file member - :rtype: List[str] + :rtype: List[bytes] """ members, members_contents = _generic_create_archive_( - archive_path, lambda: tarfile.open(archive_path, "w:gz") + lambda: tarfile.open(archive_path, "w:gz") ) return members, members_contents -def create_in_memory_fake_archive(archive_path: Path): +def create_in_memory_fake_archive() -> Tuple[bytes, List[str], List[bytes]]: with io.BytesIO() as buffer: members, members_contents = _generic_create_archive_( - archive_path, lambda: tarfile.open(fileobj=buffer, mode="w:gz") + lambda: tarfile.open(fileobj=buffer, mode="w:gz") ) archive_as_bytes = buffer.getvalue() return archive_as_bytes, members, members_contents -class TarGzFileStreamer: - def __init__(self, data, chunk_size=128): +class DataStreamer: + def __init__(self, data: bytes, chunk_size: int = 128): self.data = data self.chunk_size = chunk_size self.size = len(data) - def __iter__(self): + def __iter__(self) -> Generator[None, bytes, None]: for i in range(0, len(self.data), self.chunk_size): yield self.data[i : i + self.chunk_size] -def get_in_memory_fake_archive_stream(archive_path): - archive_as_bytes, members, members_contents = create_in_memory_fake_archive( - archive_path - ) - return TarGzFileStreamer(archive_as_bytes), members, members_contents +def get_in_memory_fake_archive_stream(): + archive_as_bytes, members, members_contents = create_in_memory_fake_archive() + return DataStreamer(archive_as_bytes), members, members_contents def get_fake_tmp_file_class(file_path: Path): @@ -133,3 +131,7 @@ def __exit__(self, exc_type, exc_value, exc_tb): self.opened_file.close() return TmpFile + + +def raise_tarfile_readerror(*args): + raise tarfile.ReadError() From 4e50501cf855501ff00a1c5acd0b5a68338efb88 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 13:52:32 +0100 Subject: [PATCH 09/33] remove double lines --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 767b10c..601c3ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,4 +27,3 @@ requests = "^2.31.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" coverage = "^7.3.2" - From 1020618058c68388678fed1bd056d379fac084a2 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 13:52:41 +0100 Subject: [PATCH 10/33] add tests workflows --- .github/workflows/pytest.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..c6d649d --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,32 @@ +name: pytest + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + max-parallel: 3 + matrix: + python-version: [3.10] + + steps: + - uses: actions/checkout@v1 + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: ${{ matrix.python-version }} + auto-update-conda: true + - name: Install dependencies and package + shell: bash --login {0} + run: | + conda info + python3 -m pip install --upgrade pip + pip install poetry + poetry install --without dev + - name: Test with pytest + shell: bash --login {0} + run: | + poetry install + python3 -m pytest --cov=src tests \ No newline at end of file From 0687e938448b39b0ba054b67b06ca27168b3cad9 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 14:47:02 +0100 Subject: [PATCH 11/33] modify workflow to use poetry --- .github/workflows/pytest.yml | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c6d649d..5447928 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,32 +1,27 @@ -name: pytest +name: Unit tests on: [push] jobs: build: - runs-on: ubuntu-latest strategy: max-parallel: 3 matrix: + os: [ubuntu-latest, ubuntu-22.04, macos-latest, windows-latest] python-version: [3.10] - + poetry-version: ["1.7.1"] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v1 - - name: Setup Conda - uses: conda-incubator/setup-miniconda@v2 - with: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: python-version: ${{ matrix.python-version }} - auto-update-conda: true - - name: Install dependencies and package - shell: bash --login {0} - run: | - conda info - python3 -m pip install --upgrade pip - pip install poetry - poetry install --without dev - - name: Test with pytest - shell: bash --login {0} - run: | - poetry install - python3 -m pytest --cov=src tests \ No newline at end of file + - name: Install poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: poetry install --with dev + - name: Run Tests + run: python3 -m pytest --cov=src tests From 6a411a8f96c874f02ce55912eb8ac3b27cba5084 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 14:50:09 +0100 Subject: [PATCH 12/33] fix target python version --- .github/workflows/{pytest.yml => unit_test.yml} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename .github/workflows/{pytest.yml => unit_test.yml} (78%) diff --git a/.github/workflows/pytest.yml b/.github/workflows/unit_test.yml similarity index 78% rename from .github/workflows/pytest.yml rename to .github/workflows/unit_test.yml index 5447928..0779845 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/unit_test.yml @@ -9,12 +9,14 @@ jobs: max-parallel: 3 matrix: os: [ubuntu-latest, ubuntu-22.04, macos-latest, windows-latest] - python-version: [3.10] + python-version: ["3.10"] poetry-version: ["1.7.1"] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - name: Checkout code + uses: actions/checkout@v4 + - name: Install python + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install poetry From c3ac8bc03108d943090d6881270b3c5612f5db34 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 14:55:05 +0100 Subject: [PATCH 13/33] update poetry lock --- poetry.lock | 129 ++++++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 76 insertions(+), 55 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6654c3f..38da3da 100644 --- a/poetry.lock +++ b/poetry.lock @@ -212,65 +212,68 @@ test-no-images = ["pytest", "pytest-cov", "wurlitzer"] [[package]] name = "coverage" -version = "7.3.2" +version = "7.3.3" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf"}, - {file = "coverage-7.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c"}, - {file = "coverage-7.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148"}, - {file = "coverage-7.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9"}, - {file = "coverage-7.3.2-cp310-cp310-win32.whl", hash = "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f"}, - {file = "coverage-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c"}, - {file = "coverage-7.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3"}, - {file = "coverage-7.3.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c"}, - {file = "coverage-7.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312"}, - {file = "coverage-7.3.2-cp311-cp311-win32.whl", hash = "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640"}, - {file = "coverage-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836"}, - {file = "coverage-7.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4"}, - {file = "coverage-7.3.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84"}, - {file = "coverage-7.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a"}, - {file = "coverage-7.3.2-cp312-cp312-win32.whl", hash = "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb"}, - {file = "coverage-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738"}, - {file = "coverage-7.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c"}, - {file = "coverage-7.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901"}, - {file = "coverage-7.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76"}, - {file = "coverage-7.3.2-cp38-cp38-win32.whl", hash = "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92"}, - {file = "coverage-7.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce"}, - {file = "coverage-7.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25"}, - {file = "coverage-7.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc"}, - {file = "coverage-7.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083"}, - {file = "coverage-7.3.2-cp39-cp39-win32.whl", hash = "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce"}, - {file = "coverage-7.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f"}, - {file = "coverage-7.3.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637"}, - {file = "coverage-7.3.2.tar.gz", hash = "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef"}, + {file = "coverage-7.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d874434e0cb7b90f7af2b6e3309b0733cde8ec1476eb47db148ed7deeb2a9494"}, + {file = "coverage-7.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6621dccce8af666b8c4651f9f43467bfbf409607c604b840b78f4ff3619aeb"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1367aa411afb4431ab58fd7ee102adb2665894d047c490649e86219327183134"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f0f8f0c497eb9c9f18f21de0750c8d8b4b9c7000b43996a094290b59d0e7523"}, + {file = "coverage-7.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0338c4b0951d93d547e0ff8d8ea340fecf5885f5b00b23be5aa99549e14cfd"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d31650d313bd90d027f4be7663dfa2241079edd780b56ac416b56eebe0a21aab"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9437a4074b43c177c92c96d051957592afd85ba00d3e92002c8ef45ee75df438"}, + {file = "coverage-7.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9e17d9cb06c13b4f2ef570355fa45797d10f19ca71395910b249e3f77942a837"}, + {file = "coverage-7.3.3-cp310-cp310-win32.whl", hash = "sha256:eee5e741b43ea1b49d98ab6e40f7e299e97715af2488d1c77a90de4a663a86e2"}, + {file = "coverage-7.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:593efa42160c15c59ee9b66c5f27a453ed3968718e6e58431cdfb2d50d5ad284"}, + {file = "coverage-7.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c944cf1775235c0857829c275c777a2c3e33032e544bcef614036f337ac37bb"}, + {file = "coverage-7.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:eda7f6e92358ac9e1717ce1f0377ed2b9320cea070906ece4e5c11d172a45a39"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c854c1d2c7d3e47f7120b560d1a30c1ca221e207439608d27bc4d08fd4aeae8"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:222b038f08a7ebed1e4e78ccf3c09a1ca4ac3da16de983e66520973443b546bc"}, + {file = "coverage-7.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff4800783d85bff132f2cc7d007426ec698cdce08c3062c8d501ad3f4ea3d16c"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fc200cec654311ca2c3f5ab3ce2220521b3d4732f68e1b1e79bef8fcfc1f2b97"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:307aecb65bb77cbfebf2eb6e12009e9034d050c6c69d8a5f3f737b329f4f15fb"}, + {file = "coverage-7.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ffb0eacbadb705c0a6969b0adf468f126b064f3362411df95f6d4f31c40d31c1"}, + {file = "coverage-7.3.3-cp311-cp311-win32.whl", hash = "sha256:79c32f875fd7c0ed8d642b221cf81feba98183d2ff14d1f37a1bbce6b0347d9f"}, + {file = "coverage-7.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:243576944f7c1a1205e5cd658533a50eba662c74f9be4c050d51c69bd4532936"}, + {file = "coverage-7.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a2ac4245f18057dfec3b0074c4eb366953bca6787f1ec397c004c78176a23d56"}, + {file = "coverage-7.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9191be7af41f0b54324ded600e8ddbcabea23e1e8ba419d9a53b241dece821d"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31c0b1b8b5a4aebf8fcd227237fc4263aa7fa0ddcd4d288d42f50eff18b0bac4"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee453085279df1bac0996bc97004771a4a052b1f1e23f6101213e3796ff3cb85"}, + {file = "coverage-7.3.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1191270b06ecd68b1d00897b2daddb98e1719f63750969614ceb3438228c088e"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:007a7e49831cfe387473e92e9ff07377f6121120669ddc39674e7244350a6a29"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:af75cf83c2d57717a8493ed2246d34b1f3398cb8a92b10fd7a1858cad8e78f59"}, + {file = "coverage-7.3.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:811ca7373da32f1ccee2927dc27dc523462fd30674a80102f86c6753d6681bc6"}, + {file = "coverage-7.3.3-cp312-cp312-win32.whl", hash = "sha256:733537a182b5d62184f2a72796eb6901299898231a8e4f84c858c68684b25a70"}, + {file = "coverage-7.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:e995efb191f04b01ced307dbd7407ebf6e6dc209b528d75583277b10fd1800ee"}, + {file = "coverage-7.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fbd8a5fe6c893de21a3c6835071ec116d79334fbdf641743332e442a3466f7ea"}, + {file = "coverage-7.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:50c472c1916540f8b2deef10cdc736cd2b3d1464d3945e4da0333862270dcb15"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e9223a18f51d00d3ce239c39fc41410489ec7a248a84fab443fbb39c943616c"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f501e36ac428c1b334c41e196ff6bd550c0353c7314716e80055b1f0a32ba394"}, + {file = "coverage-7.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:475de8213ed95a6b6283056d180b2442eee38d5948d735cd3d3b52b86dd65b92"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afdcc10c01d0db217fc0a64f58c7edd635b8f27787fea0a3054b856a6dff8717"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fff0b2f249ac642fd735f009b8363c2b46cf406d3caec00e4deeb79b5ff39b40"}, + {file = "coverage-7.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a1f76cfc122c9e0f62dbe0460ec9cc7696fc9a0293931a33b8870f78cf83a327"}, + {file = "coverage-7.3.3-cp38-cp38-win32.whl", hash = "sha256:757453848c18d7ab5d5b5f1827293d580f156f1c2c8cef45bfc21f37d8681069"}, + {file = "coverage-7.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ad2453b852a1316c8a103c9c970db8fbc262f4f6b930aa6c606df9b2766eee06"}, + {file = "coverage-7.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b15e03b8ee6a908db48eccf4e4e42397f146ab1e91c6324da44197a45cb9132"}, + {file = "coverage-7.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:89400aa1752e09f666cc48708eaa171eef0ebe3d5f74044b614729231763ae69"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c59a3e59fb95e6d72e71dc915e6d7fa568863fad0a80b33bc7b82d6e9f844973"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ede881c7618f9cf93e2df0421ee127afdfd267d1b5d0c59bcea771cf160ea4a"}, + {file = "coverage-7.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3bfd2c2f0e5384276e12b14882bf2c7621f97c35320c3e7132c156ce18436a1"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7f3bad1a9313401ff2964e411ab7d57fb700a2d5478b727e13f156c8f89774a0"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:65d716b736f16e250435473c5ca01285d73c29f20097decdbb12571d5dfb2c94"}, + {file = "coverage-7.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a702e66483b1fe602717020a0e90506e759c84a71dbc1616dd55d29d86a9b91f"}, + {file = "coverage-7.3.3-cp39-cp39-win32.whl", hash = "sha256:7fbf3f5756e7955174a31fb579307d69ffca91ad163467ed123858ce0f3fd4aa"}, + {file = "coverage-7.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cad9afc1644b979211989ec3ff7d82110b2ed52995c2f7263e7841c846a75348"}, + {file = "coverage-7.3.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:d299d379b676812e142fb57662a8d0d810b859421412b4d7af996154c00c31bb"}, + {file = "coverage-7.3.3.tar.gz", hash = "sha256:df04c64e58df96b4427db8d0559e95e2df3138c9916c96f9f6a4dd220db2fdb7"}, ] +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + [package.extras] toml = ["tomli"] @@ -812,6 +815,24 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1039,4 +1060,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "1a4a705b2fb79214bb179e9d08d8e6873bf7dcb990aa18a842c7656ebb958b43" +content-hash = "4c5285bb8e56071b093b03e3e5d3a9d4fc0ab138f1c12233da4ce9a8a5ddea1b" diff --git a/pyproject.toml b/pyproject.toml index 601c3ea..4943254 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,4 +26,4 @@ requests = "^2.31.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" -coverage = "^7.3.2" +pytest-cov = "^4.1.0" From 8578a7ac12baa52569a8e69e44c81b6cc531e3aa Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 15:00:57 +0100 Subject: [PATCH 14/33] test commands in workflow --- .github/workflows/unit_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 0779845..e5d97af 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -26,4 +26,7 @@ jobs: - name: Install dependencies run: poetry install --with dev - name: Run Tests - run: python3 -m pytest --cov=src tests + run: | + which python3 + pip list + python3 -m pytest --cov=src tests From 8b4aa9b8182f61490d46a53b24071e137c151975 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 15:08:10 +0100 Subject: [PATCH 15/33] use poetry to run pytest --- .github/workflows/unit_test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index e5d97af..cb5a683 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -26,7 +26,4 @@ jobs: - name: Install dependencies run: poetry install --with dev - name: Run Tests - run: | - which python3 - pip list - python3 -m pytest --cov=src tests + run: poetry run python -m pytest --cov=src tests From ef72aead645c1249fbdf76c40ab971d2e0fc7480 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 15:16:11 +0100 Subject: [PATCH 16/33] do not test on windows --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index cb5a683..4f21205 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 3 matrix: - os: [ubuntu-latest, ubuntu-22.04, macos-latest, windows-latest] + os: [ubuntu-latest, ubuntu-22.04, macos-latest] python-version: ["3.10"] poetry-version: ["1.7.1"] runs-on: ${{ matrix.os }} From 13774dd86c0dcb80edf25827af27e97aa562e00c Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 15:55:16 +0100 Subject: [PATCH 17/33] publish to pypi workflow --- .github/workflows/publish.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..3c456ac --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,29 @@ +name: Build wheels and deploy + +on: + workflow_dispatch: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: "1.7.1" + - name: Install dependencies + run: poetry install --with dev + - name: Package the distribution + run: poetry build + - name: Publish distribution to PyPI + if: ${{ github.event_name == 'release' && github.event.action == 'published'}} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} \ No newline at end of file From bb482f5939c6f250f65c131361625ae7c5b84c7f Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 15:58:08 +0100 Subject: [PATCH 18/33] remove unuseful name --- .github/workflows/unit_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 4f21205..51d068f 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -13,8 +13,7 @@ jobs: poetry-version: ["1.7.1"] runs-on: ${{ matrix.os }} steps: - - name: Checkout code - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v4 with: From af2554dc418bfa14ce555ead13224435ca58170f Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 17:31:27 +0100 Subject: [PATCH 19/33] add requests timeouts and do some linting --- src/qibo_tii_provider/__init__.py | 1 + src/qibo_tii_provider/config.py | 3 +++ src/qibo_tii_provider/tiiprovider.py | 40 +++++++++++++++++----------- tests/test_tiiqprovider.py | 38 +++++++++++++++++--------- tests/utils_test_tiiqprovider.py | 2 +- 5 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/qibo_tii_provider/__init__.py b/src/qibo_tii_provider/__init__.py index 664a1be..e0b9f27 100644 --- a/src/qibo_tii_provider/__init__.py +++ b/src/qibo_tii_provider/__init__.py @@ -1 +1,2 @@ +"""The `qibo_tii_provider` package""" from qibo_tii_provider.tiiprovider import TIIProvider diff --git a/src/qibo_tii_provider/config.py b/src/qibo_tii_provider/config.py index 890202f..cf938f3 100644 --- a/src/qibo_tii_provider/config.py +++ b/src/qibo_tii_provider/config.py @@ -1,3 +1,6 @@ +"""This module implements some constants and custom exceptions""" + + class MalformedResponseError(Exception): """Exception raised when server responsed body does not contain expected keys""" diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tiiprovider.py index 09c9dd7..aa73d0e 100644 --- a/src/qibo_tii_provider/tiiprovider.py +++ b/src/qibo_tii_provider/tiiprovider.py @@ -1,16 +1,17 @@ -from pathlib import Path +"""The module implementing the TIIProvider class.""" +import logging +import os import tarfile import tempfile import time +from pathlib import Path from typing import Iterable, List, Optional -import os import numpy as np import qibo import requests -from .config import MalformedResponseError, JobPostServerError - +from .config import JobPostServerError, MalformedResponseError QRCCLUSTER_IP = os.environ.get("QRCCLUSTER_IP", "login.qrccluster.com") QRCCLUSTER_PORT = os.environ.get("QRCCLUSTER_PORT", "8010") @@ -22,13 +23,14 @@ RESULTS_BASE_FOLDER = Path(RESULTS_BASE_FOLDER) RESULTS_BASE_FOLDER.mkdir(exist_ok=True) -# configure logger -import logging +TIMEOUT = 10 + +# configure logger logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(message)s") logger = logging.getLogger(__name__) -logging_level = logging.INFO -logger.setLevel(logging_level) +LOGGING_LEVEL = logging.INFO +logger.setLevel(LOGGING_LEVEL) def wait_for_response_to_get_request(url: str) -> requests.models.Response: @@ -41,7 +43,7 @@ def wait_for_response_to_get_request(url: str) -> requests.models.Response: :rtype: requests.models.Response """ while True: - response = requests.get(url) + response = requests.get(url, timeout=TIMEOUT) if response.content == b"Job still in progress": time.sleep(SECONDS_BETWEEN_CHECKS) continue @@ -118,6 +120,10 @@ def __init__(self, token: str): """ self.token = token + self.pid = None + self.results_folder = None + self.results_path = None + self.check_client_server_qibo_versions() def check_client_server_qibo_versions(self): @@ -126,15 +132,17 @@ def check_client_server_qibo_versions(self): Raise assertion error if the two versions are not the same. """ url = BASE_URL + "qibo_version/" - response = requests.get(url) + response = requests.get(url, timeout=TIMEOUT) response.raise_for_status() check_response_has_keys(response, ["qibo_version"]) qibo_server_version = response.json()["qibo_version"] qibo_local_version = qibo.__version__ - assert ( - qibo_local_version == qibo_server_version - ), f"Local Qibo package version does not match the server one, please upgrade: {qibo_local_version} -> {qibo_server_version}" + msg = ( + "Local Qibo package version does not match the server one, please " + f"upgrade: {qibo_local_version} -> {qibo_server_version}" + ) + assert qibo_local_version == qibo_server_version, msg def run_circuit( self, circuit: qibo.Circuit, nshots: int = 1000, device: str = "sim" @@ -173,7 +181,7 @@ def run_circuit( self._post_circuit(circuit, nshots, device) except JobPostServerError as err: logger.error(err.message) - return + return None # retrieve results logger.info("Job posted on server with pid %s", self.pid) @@ -193,7 +201,7 @@ def _post_circuit( "nshots": nshots, "device": device, } - response = requests.post(url, json=payload) + response = requests.post(url, json=payload, timeout=TIMEOUT) # checks response.raise_for_status() @@ -236,7 +244,7 @@ def _get_result(self) -> Optional[np.ndarray]: "the file at `%s`", self.results_folder.as_posix(), ) - return + return None if response.headers["Job-Status"].lower() == "error": logger.info( diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index fbd787d..6c28fb2 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -1,21 +1,21 @@ -from pathlib import Path import tarfile +from pathlib import Path from typing import Callable -from unittest.mock import call, patch, Mock +from unittest.mock import Mock, call, patch import pytest from requests.exceptions import HTTPError import tests.utils_test_tiiqprovider as utils - from qibo_tii_provider import tiiprovider -from qibo_tii_provider.config import MalformedResponseError, JobPostServerError +from qibo_tii_provider.config import JobPostServerError, MalformedResponseError PKG = "qibo_tii_provider.tiiprovider" LOCAL_URL = "http://localhost:8000/" FAKE_QIBO_VERSION = "0.0.1" FAKE_PID = "123" ARCHIVE_NAME = "file.tar.gz" +TIMEOUT = 1 @pytest.fixture(autouse=True) @@ -34,6 +34,13 @@ def mock_qibo(): yield _mock_qibo +@pytest.fixture(scope="module", autouse=True) +def mock_timeout(): + """Ensure that all the requests are made on localhost""" + with patch(f"{PKG}.TIMEOUT", TIMEOUT) as _fixture: + yield _fixture + + @pytest.fixture def results_base_folder(tmp_path: Path): results_base_folder = tmp_path / "results" @@ -55,10 +62,11 @@ def _get_request_side_effect(job_status: str = "success") -> Callable: :rtype: Callable """ - def _request_side_effect(url): + def _request_side_effect(url, timeout): if url == LOCAL_URL + "qibo_version/": return utils.MockedResponse( - status_code=200, json_data={"qibo_version": FAKE_QIBO_VERSION} + status_code=200, + json_data={"qibo_version": FAKE_QIBO_VERSION}, ) if url == LOCAL_URL + f"get_result/{FAKE_PID}/": stream, _, _ = utils.get_in_memory_fake_archive_stream() @@ -72,7 +80,7 @@ def _request_side_effect(url): return _request_side_effect -def _post_request_side_effect(url, json): +def _post_request_side_effect(url, json, timeout): if url == LOCAL_URL + "run_circuit/": json_data = {"pid": FAKE_PID, "message": "Success. Job posted"} return utils.MockedResponse(status_code=200, json_data=json_data) @@ -123,13 +131,15 @@ def _get_tii_client(): def test_check_client_server_qibo_versions_with_version_match(mock_request: Mock): _get_tii_client() - mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") + mock_request.get.assert_called_once_with( + LOCAL_URL + "qibo_version/", timeout=TIMEOUT + ) def test_check_client_server_qibo_versions_with_version_mismatch(mock_request: Mock): remote_qibo_version = "0.2.2" - def _new_side_effect(url): + def _new_side_effect(url, timeout): return utils.MockedResponse( status_code=200, json_data={"qibo_version": remote_qibo_version} ) @@ -139,11 +149,13 @@ def _new_side_effect(url): with pytest.raises(AssertionError): _get_tii_client() - mock_request.get.assert_called_once_with(LOCAL_URL + "qibo_version/") + mock_request.get.assert_called_once_with( + LOCAL_URL + "qibo_version/", timeout=TIMEOUT + ) def test__post_circuit_with_invalid_token(mock_request: Mock): - def _new_side_effect(url, json): + def _new_side_effect(url, json, timeout): return utils.MockedResponse(status_code=404) mock_request.post.side_effect = _new_side_effect @@ -154,7 +166,7 @@ def _new_side_effect(url, json): def test__post_circuit_not_successful(mock_request: Mock): - def _new_side_effect(url, json): + def _new_side_effect(url, json, timeout): json_data = {"pid": None, "message": "post job to queue failed"} return utils.MockedResponse(status_code=200, json_data=json_data) @@ -166,7 +178,7 @@ def _new_side_effect(url, json): def test__run_circuit_with_unsuccessful_post_to_queue(mock_request: Mock): - def _new_side_effect(url, json): + def _new_side_effect(url, json, timeout): json_data = {"pid": None, "message": "post job to queue failed"} return utils.MockedResponse(status_code=200, json_data=json_data) diff --git a/tests/utils_test_tiiqprovider.py b/tests/utils_test_tiiqprovider.py index d0c94e1..249bb62 100644 --- a/tests/utils_test_tiiqprovider.py +++ b/tests/utils_test_tiiqprovider.py @@ -1,6 +1,6 @@ import io -from pathlib import Path import tarfile +from pathlib import Path from typing import Dict, Generator, List, Optional, Tuple from requests.exceptions import HTTPError From 0712d5e9d9a2596285412398a754008e0a03b215 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 17:51:21 +0100 Subject: [PATCH 20/33] conform to qibo workflow policies --- .github/workflows/deploy.yml | 24 ++++++++++++++++++++ .github/workflows/publish.yml | 29 ------------------------ .github/workflows/unit_test.yml | 39 ++++++++++++++------------------- pyproject.toml | 17 ++++++++++++++ 4 files changed, 58 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/deploy.yml delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..d50535a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,24 @@ +# A single CI script with github workflow +name: Build wheels and deploy + +on: + workflow_dispatch: + push: + merge_group: + release: + types: + - published + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ['3.10', '3.11'] + uses: qiboteam/workflows/.github/workflows/deploy-pip-poetry.yml@main + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + publish: ${{ github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} + poetry-extras: "--with tests,docs" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 3c456ac..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build wheels and deploy - -on: - workflow_dispatch: - release: - types: [published] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install poetry - uses: abatilo/actions-poetry@v2 - with: - poetry-version: "1.7.1" - - name: Install dependencies - run: poetry install --with dev - - name: Package the distribution - run: poetry build - - name: Publish distribution to PyPI - if: ${{ github.event_name == 'release' && github.event.action == 'published'}} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 51d068f..71a50f5 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -1,28 +1,23 @@ -name: Unit tests +name: Tests -on: [push] +on: + workflow_dispatch: + push: + merge_group: + pull_request: + types: [labeled] jobs: build: - + if: contains(github.event.pull_request.labels.*.name, 'run-workflow') || github.event_name == 'push' strategy: - max-parallel: 3 matrix: - os: [ubuntu-latest, ubuntu-22.04, macos-latest] - python-version: ["3.10"] - poetry-version: ["1.7.1"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Install python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install poetry - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} - - name: Install dependencies - run: poetry install --with dev - - name: Run Tests - run: poetry run python -m pytest --cov=src tests + os: [ubuntu-latest, macos-latest] + python-version: ['3.10', '3.11'] + uses: qiboteam/workflows/.github/workflows/rules-poetry.yml@main + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + doctests: ${{ matrix.os == 'ubuntu-latest'}} + poetry-extras: "--with tests,docs" + secrets: inherit \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4943254..6b2feed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,3 +27,20 @@ requests = "^2.31.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" pytest-cov = "^4.1.0" + +[tool.poe.tasks] +test = "python -m pytest cov=src tests" +lint = "pylint --errors-only src" +lint-warnings = "pylint --exit-zero src" + +[tool.pylint.reports] +output-format = "colorized" + +[tool.pytest.ini_options] +testpaths = ['tests/'] +filterwarnings = ['ignore::RuntimeWarning'] +addopts = [ + '--cov=src/qibo_tii_provider', + '--cov-report=xml', + '--cov-report=html', +] \ No newline at end of file From 8a84bdb71a0f9323610208ab514b1e5e61829926 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 17:51:40 +0100 Subject: [PATCH 21/33] remove unused import --- tests/test_tiiqprovider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index 6c28fb2..9ec348d 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -1,12 +1,12 @@ import tarfile from pathlib import Path from typing import Callable -from unittest.mock import Mock, call, patch +from unittest.mock import Mock, patch import pytest from requests.exceptions import HTTPError -import tests.utils_test_tiiqprovider as utils +import utils_test_tiiqprovider as utils from qibo_tii_provider import tiiprovider from qibo_tii_provider.config import JobPostServerError, MalformedResponseError From a6b9b0a5ae693a0d6571e00e0387430e1325b34b Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 17:55:13 +0100 Subject: [PATCH 22/33] fix poe test task command --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6b2feed..4287cac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ pytest = "^7.4.3" pytest-cov = "^4.1.0" [tool.poe.tasks] -test = "python -m pytest cov=src tests" +test = "pytest" lint = "pylint --errors-only src" lint-warnings = "pylint --exit-zero src" From 0ddb793905bb3e1dfa4982d97c2438152adc8566 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 17:56:14 +0100 Subject: [PATCH 23/33] fix workflow poetry extra groups --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d50535a..ff4ab71 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,5 +20,5 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} publish: ${{ github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} - poetry-extras: "--with tests,docs" + poetry-extras: "--with dev" secrets: inherit \ No newline at end of file From 788ff238bc569f36f8b5240a150ebab6a724c35a Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 18:00:53 +0100 Subject: [PATCH 24/33] reduce matrix --- .github/workflows/deploy.yml | 2 +- .github/workflows/unit_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ff4ab71..54132a5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.10', '3.11'] + python-version: ['3.10'] uses: qiboteam/workflows/.github/workflows/deploy-pip-poetry.yml@main with: os: ${{ matrix.os }} diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 71a50f5..e72b2f1 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.10', '3.11'] + python-version: ['3.10'] uses: qiboteam/workflows/.github/workflows/rules-poetry.yml@main with: os: ${{ matrix.os }} From 5ccfd8f49b1d633e2335d2f0435499be60b584a4 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 18:03:45 +0100 Subject: [PATCH 25/33] fix poetry group called in workflow --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index e72b2f1..63054c9 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -19,5 +19,5 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} doctests: ${{ matrix.os == 'ubuntu-latest'}} - poetry-extras: "--with tests,docs" + poetry-extras: "--with dev" secrets: inherit \ No newline at end of file From b849d1322171fdaddfc74592ae26b6ff42195ee6 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 18:09:34 +0100 Subject: [PATCH 26/33] add pylint to venv --- poetry.lock | 110 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 38da3da..355e55b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,19 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "astroid" +version = "3.0.2" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "astroid-3.0.2-py3-none-any.whl", hash = "sha256:d6e62862355f60e716164082d6b4b041d38e2a8cf1c7cd953ded5108bac8ff5c"}, + {file = "astroid-3.0.2.tar.gz", hash = "sha256:4a61cf0a59097c7bb52689b0fd63717cd2a8a14dc9f1eee97b82d814881c8c91"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} + [[package]] name = "certifi" version = "2023.7.22" @@ -292,6 +306,20 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "dill" +version = "0.3.7" +description = "serialize all of Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e"}, + {file = "dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -411,6 +439,20 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "joblib" version = "1.3.2" @@ -604,6 +646,17 @@ pyparsing = ">=2.3.1" python-dateutil = ">=2.7" setuptools_scm = ">=7" +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -736,6 +789,21 @@ files = [ docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "platformdirs" +version = "4.1.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + [[package]] name = "pluggy" version = "1.3.0" @@ -779,6 +847,35 @@ files = [ [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "pylint" +version = "3.0.3" +description = "python code static checker" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "pylint-3.0.3-py3-none-any.whl", hash = "sha256:7a1585285aefc5165db81083c3e06363a27448f6b467b3b0f30dbd0ac1f73810"}, + {file = "pylint-3.0.3.tar.gz", hash = "sha256:58c2398b0301e049609a8429789ec6edf3aabe9b6c5fec916acd18639c16de8b"}, +] + +[package.dependencies] +astroid = ">=3.0.1,<=3.1.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, +] +isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" +typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + [[package]] name = "pyparsing" version = "3.1.1" @@ -1014,6 +1111,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomlkit" +version = "0.12.3" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" @@ -1060,4 +1168,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "4c5285bb8e56071b093b03e3e5d3a9d4fc0ab138f1c12233da4ce9a8a5ddea1b" +content-hash = "50db5074b7fa12d380bdb916c5bb9119f1acc11f71002786c5868e809b5da86a" diff --git a/pyproject.toml b/pyproject.toml index 4287cac..60e82de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ requests = "^2.31.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" pytest-cov = "^4.1.0" +pylint = "^3.0.3" [tool.poe.tasks] test = "pytest" From 8857ec1e17399a69f80a420db8070732e8dfa463 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Thu, 14 Dec 2023 18:13:22 +0100 Subject: [PATCH 27/33] do not build docs in CI CD --- .github/workflows/unit_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 63054c9..86cd790 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -18,6 +18,5 @@ jobs: with: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} - doctests: ${{ matrix.os == 'ubuntu-latest'}} poetry-extras: "--with dev" secrets: inherit \ No newline at end of file From b5d68c91e745b8e6041472231bf0195cfbbe50ed Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 15 Dec 2023 09:41:06 +0100 Subject: [PATCH 28/33] test on more python versions --- .github/workflows/deploy.yml | 2 +- .github/workflows/unit_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 54132a5..6f1ea52 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.10'] + python-version: [3.9, '3.10', '3.11'] uses: qiboteam/workflows/.github/workflows/deploy-pip-poetry.yml@main with: os: ${{ matrix.os }} diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 86cd790..7ffa335 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.10'] + python-version: [3.9, '3.10', '3.11'] uses: qiboteam/workflows/.github/workflows/rules-poetry.yml@main with: os: ${{ matrix.os }} From 06f91d8409e42adb75f4ec5a2887e74f3838e115 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 08:41:16 +0000 Subject: [PATCH 29/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/deploy.yml | 2 +- .github/workflows/unit_test.yml | 2 +- .pre-commit-config.yaml | 2 +- examples/run_error_job.py | 3 ++- examples/run_successful_job.py | 3 ++- pyproject.toml | 2 +- tests/test_tiiqprovider.py | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6f1ea52..e2fb354 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,4 +21,4 @@ jobs: python-version: ${{ matrix.python-version }} publish: ${{ github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} poetry-extras: "--with dev" - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 7ffa335..8e424f7 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -19,4 +19,4 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} poetry-extras: "--with dev" - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 128773f..2d641a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,4 +25,4 @@ repos: rev: v2.4.0 hooks: - id: pycln - args: [--config=pyproject.toml] \ No newline at end of file + args: [--config=pyproject.toml] diff --git a/examples/run_error_job.py b/examples/run_error_job.py index c033fce..5f28689 100644 --- a/examples/run_error_job.py +++ b/examples/run_error_job.py @@ -1,11 +1,12 @@ import qibo + from qibo_tii_provider import TiiProvider # create the circuit you want to run circuit = qibo.models.QFT(11) # read the token from file -with open("token.txt", "r") as f: +with open("token.txt") as f: token = f.read() # authenticate to server through the client instance diff --git a/examples/run_successful_job.py b/examples/run_successful_job.py index 8efe925..0f79dff 100644 --- a/examples/run_successful_job.py +++ b/examples/run_successful_job.py @@ -1,11 +1,12 @@ import qibo + from qibo_tii_provider import TIIProvider # create the circuit you want to run circuit = qibo.models.QFT(5) # read the token from file -with open("token.txt", "r") as f: +with open("token.txt") as f: token = f.read() # authenticate to server through the client instance diff --git a/pyproject.toml b/pyproject.toml index 60e82de..e82212d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,4 +44,4 @@ addopts = [ '--cov=src/qibo_tii_provider', '--cov-report=xml', '--cov-report=html', -] \ No newline at end of file +] diff --git a/tests/test_tiiqprovider.py b/tests/test_tiiqprovider.py index 9ec348d..955937f 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tiiqprovider.py @@ -4,9 +4,9 @@ from unittest.mock import Mock, patch import pytest +import utils_test_tiiqprovider as utils from requests.exceptions import HTTPError -import utils_test_tiiqprovider as utils from qibo_tii_provider import tiiprovider from qibo_tii_provider.config import JobPostServerError, MalformedResponseError From 06c53aecd96efd3a11ba5de61a621b468af0ca2e Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 15 Dec 2023 10:14:41 +0100 Subject: [PATCH 30/33] rename ttiq_provider -> tii_qrc_provider --- .gitignore | 2 +- src/qibo_tii_provider/__init__.py | 2 +- .../{tiiprovider.py => tii_qrc_provider.py} | 0 ...iqprovider.py => test_tii_qrc_provider.py} | 24 +++++++++---------- ...ider.py => utils_test_tii_qrc_provider.py} | 0 5 files changed, 14 insertions(+), 14 deletions(-) rename src/qibo_tii_provider/{tiiprovider.py => tii_qrc_provider.py} (100%) rename tests/{test_tiiqprovider.py => test_tii_qrc_provider.py} (92%) rename tests/{utils_test_tiiqprovider.py => utils_test_tii_qrc_provider.py} (100%) diff --git a/.gitignore b/.gitignore index 66614b1..de63fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -164,4 +164,4 @@ examples/token.txt examples/results/ # configuration variables -src/tiiq_provider/.env +src/tii_qrc_provider/.env diff --git a/src/qibo_tii_provider/__init__.py b/src/qibo_tii_provider/__init__.py index e0b9f27..ab9e664 100644 --- a/src/qibo_tii_provider/__init__.py +++ b/src/qibo_tii_provider/__init__.py @@ -1,2 +1,2 @@ """The `qibo_tii_provider` package""" -from qibo_tii_provider.tiiprovider import TIIProvider +from qibo_tii_provider.tii_qrc_provider import TIIProvider diff --git a/src/qibo_tii_provider/tiiprovider.py b/src/qibo_tii_provider/tii_qrc_provider.py similarity index 100% rename from src/qibo_tii_provider/tiiprovider.py rename to src/qibo_tii_provider/tii_qrc_provider.py diff --git a/tests/test_tiiqprovider.py b/tests/test_tii_qrc_provider.py similarity index 92% rename from tests/test_tiiqprovider.py rename to tests/test_tii_qrc_provider.py index 9ec348d..01c2b2d 100644 --- a/tests/test_tiiqprovider.py +++ b/tests/test_tii_qrc_provider.py @@ -6,11 +6,11 @@ import pytest from requests.exceptions import HTTPError -import utils_test_tiiqprovider as utils -from qibo_tii_provider import tiiprovider +import utils_test_tii_qrc_provider as utils +from qibo_tii_provider import tii_qrc_provider from qibo_tii_provider.config import JobPostServerError, MalformedResponseError -PKG = "qibo_tii_provider.tiiprovider" +PKG = "qibo_tii_provider.tii_qrc_provider" LOCAL_URL = "http://localhost:8000/" FAKE_QIBO_VERSION = "0.0.1" FAKE_PID = "123" @@ -112,7 +112,7 @@ def test_check_response_has_keys(): json_data = {"key1": 0, "key2": 1} status_code = 200 mock_response = utils.MockedResponse(status_code, json_data) - tiiprovider.check_response_has_keys(mock_response, keys) + tii_qrc_provider.check_response_has_keys(mock_response, keys) def test_check_response_has_missing_keys(): @@ -122,11 +122,11 @@ def test_check_response_has_missing_keys(): status_code = 200 mock_response = utils.MockedResponse(status_code, json_data) with pytest.raises(MalformedResponseError): - tiiprovider.check_response_has_keys(mock_response, keys) + tii_qrc_provider.check_response_has_keys(mock_response, keys) def _get_tii_client(): - return tiiprovider.TIIProvider("valid_token") + return tii_qrc_provider.TIIProvider("valid_token") def test_check_client_server_qibo_versions_with_version_match(mock_request: Mock): @@ -202,7 +202,7 @@ def test_wait_for_response_to_get_request(mock_request: Mock): mock_request.get.side_effect = [keep_waiting] * failed_attempts + [job_done] with patch(f"{PKG}.SECONDS_BETWEEN_CHECKS", 1e-4): - tiiprovider.wait_for_response_to_get_request(url) + tii_qrc_provider.wait_for_response_to_get_request(url) assert mock_request.get.call_count == failed_attempts + 1 @@ -220,7 +220,7 @@ def test__write_stream_to_tmp_file_with_simple_text_stream( assert not archive_path.is_file() - result_path = tiiprovider._write_stream_to_tmp_file(stream) + result_path = tii_qrc_provider._write_stream_to_tmp_file(stream) assert result_path == archive_path assert result_path.is_file() @@ -238,7 +238,7 @@ def test__write_stream_to_tmp_file(mock_tempfile: Mock, archive_path: Path): assert not archive_path.is_file() - result_path = tiiprovider._write_stream_to_tmp_file(stream) + result_path = tii_qrc_provider._write_stream_to_tmp_file(stream) assert result_path == archive_path assert result_path.is_file() @@ -259,7 +259,7 @@ def test__extract_archive_to_folder_with_non_archive_input(tmp_path): file_path.write_text("test content") with pytest.raises(tarfile.ReadError): - tiiprovider._extract_archive_to_folder(file_path, tmp_path) + tii_qrc_provider._extract_archive_to_folder(file_path, tmp_path) @patch( @@ -278,7 +278,7 @@ def test__get_result_handles_tarfile_readerror(mock_request, results_base_folder def test__extract_archive_to_folder(archive_path, results_base_folder): members, members_contents = utils.create_fake_archive(archive_path) - tiiprovider._extract_archive_to_folder(archive_path, results_base_folder) + tii_qrc_provider._extract_archive_to_folder(archive_path, results_base_folder) result_members = [] result_members_contents = [] @@ -297,7 +297,7 @@ def test__save_and_unpack_stream_response_to_folder( assert not archive_path.is_file() - tiiprovider._save_and_unpack_stream_response_to_folder(stream, results_base_folder) + tii_qrc_provider._save_and_unpack_stream_response_to_folder(stream, results_base_folder) # the archive should have been removed assert not archive_path.is_file() diff --git a/tests/utils_test_tiiqprovider.py b/tests/utils_test_tii_qrc_provider.py similarity index 100% rename from tests/utils_test_tiiqprovider.py rename to tests/utils_test_tii_qrc_provider.py From 4987e0ce4a889886e94a66a1fdd54a0216e96edf Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 15 Dec 2023 10:15:05 +0100 Subject: [PATCH 31/33] tiiq_provider -> tii_qrc_provider --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0db89d..dfa110a 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Please, sign up to [this link](http://http://login.qrccluster.com:8010/) to obtain the needed token to run computations on the cluster. The following snippet provides a basic usage example. -Replace the `your-tiiq-token` string with your user token received during the +Replace the `your-tii-qrc-token` string with your user token received during the registration process. ```python @@ -40,7 +40,7 @@ from qibo_tii_provider import TIIProvider circuit = qibo.models.QFT(5) # authenticate to server through the client instance -token = "your-tiiq-token" +token = "your-tii-qrc-token" client = TIIProvider(token) # run the circuit From 40c500d1cb28f3e07dfc6f8cb94886bb577fe376 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 15 Dec 2023 10:18:49 +0100 Subject: [PATCH 32/33] add newlines --- .github/workflows/deploy.yml | 2 +- .github/workflows/unit_test.yml | 2 +- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- tests/test_tii_qrc_provider.py | 4 +++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6f1ea52..e2fb354 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,4 +21,4 @@ jobs: python-version: ${{ matrix.python-version }} publish: ${{ github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} poetry-extras: "--with dev" - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 7ffa335..8e424f7 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -19,4 +19,4 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} poetry-extras: "--with dev" - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 128773f..2d641a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,4 +25,4 @@ repos: rev: v2.4.0 hooks: - id: pycln - args: [--config=pyproject.toml] \ No newline at end of file + args: [--config=pyproject.toml] diff --git a/pyproject.toml b/pyproject.toml index 60e82de..e82212d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,4 +44,4 @@ addopts = [ '--cov=src/qibo_tii_provider', '--cov-report=xml', '--cov-report=html', -] \ No newline at end of file +] diff --git a/tests/test_tii_qrc_provider.py b/tests/test_tii_qrc_provider.py index 01c2b2d..b5af100 100644 --- a/tests/test_tii_qrc_provider.py +++ b/tests/test_tii_qrc_provider.py @@ -297,7 +297,9 @@ def test__save_and_unpack_stream_response_to_folder( assert not archive_path.is_file() - tii_qrc_provider._save_and_unpack_stream_response_to_folder(stream, results_base_folder) + tii_qrc_provider._save_and_unpack_stream_response_to_folder( + stream, results_base_folder + ) # the archive should have been removed assert not archive_path.is_file() From ad4224d5936cf703afdd32ba00f011f251db662f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 09:21:27 +0000 Subject: [PATCH 33/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_tii_qrc_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tii_qrc_provider.py b/tests/test_tii_qrc_provider.py index b5af100..b7657be 100644 --- a/tests/test_tii_qrc_provider.py +++ b/tests/test_tii_qrc_provider.py @@ -4,9 +4,9 @@ from unittest.mock import Mock, patch import pytest +import utils_test_tii_qrc_provider as utils from requests.exceptions import HTTPError -import utils_test_tii_qrc_provider as utils from qibo_tii_provider import tii_qrc_provider from qibo_tii_provider.config import JobPostServerError, MalformedResponseError