diff --git a/poetry.lock b/poetry.lock index 0c2294ac..7395272f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -819,37 +819,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "debugpy" -version = "1.8.9" +version = "1.8.10" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" files = [ - {file = "debugpy-1.8.9-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:cfe1e6c6ad7178265f74981edf1154ffce97b69005212fbc90ca22ddfe3d017e"}, - {file = "debugpy-1.8.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada7fb65102a4d2c9ab62e8908e9e9f12aed9d76ef44880367bc9308ebe49a0f"}, - {file = "debugpy-1.8.9-cp310-cp310-win32.whl", hash = "sha256:c36856343cbaa448171cba62a721531e10e7ffb0abff838004701454149bc037"}, - {file = "debugpy-1.8.9-cp310-cp310-win_amd64.whl", hash = "sha256:17c5e0297678442511cf00a745c9709e928ea4ca263d764e90d233208889a19e"}, - {file = "debugpy-1.8.9-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:b74a49753e21e33e7cf030883a92fa607bddc4ede1aa4145172debc637780040"}, - {file = "debugpy-1.8.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62d22dacdb0e296966d7d74a7141aaab4bec123fa43d1a35ddcb39bf9fd29d70"}, - {file = "debugpy-1.8.9-cp311-cp311-win32.whl", hash = "sha256:8138efff315cd09b8dcd14226a21afda4ca582284bf4215126d87342bba1cc66"}, - {file = "debugpy-1.8.9-cp311-cp311-win_amd64.whl", hash = "sha256:ff54ef77ad9f5c425398efb150239f6fe8e20c53ae2f68367eba7ece1e96226d"}, - {file = "debugpy-1.8.9-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:957363d9a7a6612a37458d9a15e72d03a635047f946e5fceee74b50d52a9c8e2"}, - {file = "debugpy-1.8.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e565fc54b680292b418bb809f1386f17081d1346dca9a871bf69a8ac4071afe"}, - {file = "debugpy-1.8.9-cp312-cp312-win32.whl", hash = "sha256:3e59842d6c4569c65ceb3751075ff8d7e6a6ada209ceca6308c9bde932bcef11"}, - {file = "debugpy-1.8.9-cp312-cp312-win_amd64.whl", hash = "sha256:66eeae42f3137eb428ea3a86d4a55f28da9bd5a4a3d369ba95ecc3a92c1bba53"}, - {file = "debugpy-1.8.9-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:957ecffff80d47cafa9b6545de9e016ae8c9547c98a538ee96ab5947115fb3dd"}, - {file = "debugpy-1.8.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1efbb3ff61487e2c16b3e033bc8595aea578222c08aaf3c4bf0f93fadbd662ee"}, - {file = "debugpy-1.8.9-cp313-cp313-win32.whl", hash = "sha256:7c4d65d03bee875bcb211c76c1d8f10f600c305dbd734beaed4077e902606fee"}, - {file = "debugpy-1.8.9-cp313-cp313-win_amd64.whl", hash = "sha256:e46b420dc1bea64e5bbedd678148be512442bc589b0111bd799367cde051e71a"}, - {file = "debugpy-1.8.9-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:472a3994999fe6c0756945ffa359e9e7e2d690fb55d251639d07208dbc37caea"}, - {file = "debugpy-1.8.9-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365e556a4772d7d0d151d7eb0e77ec4db03bcd95f26b67b15742b88cacff88e9"}, - {file = "debugpy-1.8.9-cp38-cp38-win32.whl", hash = "sha256:54a7e6d3014c408eb37b0b06021366ee985f1539e12fe49ca2ee0d392d9ceca5"}, - {file = "debugpy-1.8.9-cp38-cp38-win_amd64.whl", hash = "sha256:8e99c0b1cc7bf86d83fb95d5ccdc4ad0586d4432d489d1f54e4055bcc795f693"}, - {file = "debugpy-1.8.9-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:7e8b079323a56f719977fde9d8115590cb5e7a1cba2fcee0986ef8817116e7c1"}, - {file = "debugpy-1.8.9-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6953b335b804a41f16a192fa2e7851bdcfd92173cbb2f9f777bb934f49baab65"}, - {file = "debugpy-1.8.9-cp39-cp39-win32.whl", hash = "sha256:7e646e62d4602bb8956db88b1e72fe63172148c1e25c041e03b103a25f36673c"}, - {file = "debugpy-1.8.9-cp39-cp39-win_amd64.whl", hash = "sha256:3d9755e77a2d680ce3d2c5394a444cf42be4a592caaf246dbfbdd100ffcf7ae5"}, - {file = "debugpy-1.8.9-py2.py3-none-any.whl", hash = "sha256:cc37a6c9987ad743d9c3a14fa1b1a14b7e4e6041f9dd0c8abf8895fe7a97b899"}, - {file = "debugpy-1.8.9.zip", hash = "sha256:1339e14c7d980407248f09824d1b25ff5c5616651689f1e0f0e51bdead3ea13e"}, + {file = "debugpy-1.8.10-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97aa00af95983887806e06f37e144909d35215d66db74f8b0e9799b4eef40cfd"}, + {file = "debugpy-1.8.10.tar.gz", hash = "sha256:ee4ed903cbeb14ee1839549f953af519ffa512598ec987b2051f9c868e2249a8"}, ] [[package]] @@ -2255,22 +2231,6 @@ files = [ [package.extras] cp2110 = ["hidapi"] -[[package]] -name = "pysondb-v2" -version = "2.2.0" -description = "A Simple, Lightweight, Efficent JSON based database for Python." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "pysondb-v2-2.2.0.tar.gz", hash = "sha256:1f5d2d864e5f5d1334d639681199255526ab0c8aa0b4b324462e78aae0e08d10"}, - {file = "pysondb_v2-2.2.0-py2.py3-none-any.whl", hash = "sha256:3226ee94b7550e602f7061f68ae64b5d02812153d799b57aa74343f20ed0cd9b"}, -] - -[package.extras] -all = ["prettytable (==3.3.0)", "ujson (==5.2.0)"] -pretttytable = ["prettytable (==3.3.0)"] -ujson = ["ujson (==5.2.0)"] - [[package]] name = "pytest" version = "7.4.4" @@ -3334,4 +3294,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "cca715f9b292591404a5ef3fb4e73ee85965949f05a9bc1c03e765c58d292782" +content-hash = "23bace37a05c27e70eb050db9a561e2f2f5672844a11ddd0e90fb1945b13fa35" diff --git a/pyproject.toml b/pyproject.toml index 6f9faf12..6216e8f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ pipx = "^1.1.0" pygithub = "^2.1.1" pyright = ">=1.1.341" pyserial = ">=3.5" -pysondb-v2 = "^2.1.0" pypi-simple = "^1.0.0" rich-click = "^1.8.3" tabulate = "^0.9.0" diff --git a/src/stubber/publish/database.py b/src/stubber/publish/database.py index ad7e6973..138c0dfb 100644 --- a/src/stubber/publish/database.py +++ b/src/stubber/publish/database.py @@ -1,26 +1,59 @@ -"""basic interface to the json database""" +"""Interface to the database which stores the package information""" import sqlite3 from pathlib import Path -from typing import Union -from pysondb import PysonDB - -def get_database(publish_path: Path, production: bool = False) -> sqlite3.Connection: +def get_database(db_path: Path, production: bool = False) -> sqlite3.Connection: """ Open the sqlite database at the given path. The database should be located in a subfolder `/data` of the root path. The database name is determined by the production flag as `all_packages[_test].db` """ + db_path = Path(db_path) + if db_path.stem == "publish": + db_path = db_path / ".." / "data" # go up one level to find the database - publish_path = Path(publish_path) - db_path = publish_path / f"all_packages{'' if production else '_test'}.db" - if not db_path.exists(): - raise FileNotFoundError("Database file not found") + db_path = db_path / f"all_packages{'' if production else '_test'}.db" + if db_path.exists(): + conn = sqlite3.connect(db_path) - conn = sqlite3.connect(db_path) + else: + print(FileNotFoundError(f"Database file not found in path: {db_path}")) + conn = create_database(db_path) + + conn.row_factory = sqlite3.Row # return rows as dicts return conn +def create_database(db_path: Path) -> sqlite3.Connection: + """ + Create a new database at the given path. + + The database should be located in a subfolder `/data` of the root path. + """ + db_path = Path(db_path) + if not db_path.parent.exists(): + db_path.parent.mkdir(parents=True) + conn = sqlite3.connect(db_path) + SCHEMA = """ + CREATE TABLE "packages" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + description TEXT, + mpy_version TEXT, + pkg_version TEXT, + publish BOOLEAN, + stub_sources TEXT, -- json string + path TEXT, + hash TEXT, + stub_hash TEXT, + port TEXT DEFAULT "", + board TEXT DEFAULT "", + variant TEXT DEFAULT "" + ) + """ + conn.execute(SCHEMA) + conn.commit + return conn diff --git a/src/stubber/publish/package.py b/src/stubber/publish/package.py index 425da027..07de7437 100644 --- a/src/stubber/publish/package.py +++ b/src/stubber/publish/package.py @@ -10,8 +10,7 @@ from mpflash.logger import log from mpflash.versions import clean_version -from packaging.version import parse -from pysondb import PysonDB + from stubber.publish.defaults import GENERIC, GENERIC_L, default_board from stubber.publish.enums import StubSource @@ -78,18 +77,16 @@ def get_package_info( """ # find in the database - SQL_Q = f""" - SELECT * - FROM packages - WHERE name = '{pkg_name}' AND mpy_version LIKE '{mpy_version}%' - ORDER BY pkg_version DESC - """ - log.trace(f"SQL Query: {SQL_Q}") cursor = db_conn.cursor() - cursor.execute(SQL_Q) - cursor.row_factory = sqlite3.Row # to get dict like access + cursor.execute( + """ + SELECT * FROM packages + WHERE name = ? AND mpy_version LIKE ? + ORDER BY pkg_version DESC + """, + (pkg_name, f"{mpy_version}%"), + ) packages = cursor.fetchall() - log.debug(f"Found {len(packages)} packages for {pkg_name} == {mpy_version}") if len(packages) > 0: pkg_from_db = dict(packages[-1]) diff --git a/src/stubber/publish/stubpackage.py b/src/stubber/publish/stubpackage.py index 53617d65..48614e31 100644 --- a/src/stubber/publish/stubpackage.py +++ b/src/stubber/publish/stubpackage.py @@ -25,7 +25,6 @@ from mpflash.logger import log from mpflash.versions import SET_PREVIEW, V_PREVIEW, clean_version from packaging.version import Version, parse -from pysondb import PysonDB from stubber.publish.bump import bump_version from stubber.publish.defaults import GENERIC_U, default_board @@ -216,6 +215,7 @@ def __init__( mpy_version: str = "0.0.1", port: str, board: str = GENERIC_U, + variant: Optional[str] = None, description: str = "MicroPython stubs", stubs: Optional[StubSources] = None, # json_data: Optional[Dict[str, Any]] = None, @@ -226,6 +226,7 @@ def __init__( self.mpy_version = mpy_version self.port = port self.board = board + self.variant = variant or "" self.description = description self.stub_sources = stubs or [] self.hash = None # intial hash @@ -405,6 +406,9 @@ def to_dict(self) -> dict: "description": self.description, "hash": self.hash, "stub_hash": self.stub_hash, + "port": self.port, + "board": self.board, + "variant": "", # TODO: get the variant } def from_dict(self, json_data: Dict) -> None: @@ -416,6 +420,10 @@ def from_dict(self, json_data: Dict) -> None: self._publish = json_data["publish"] self.hash = json_data["hash"] self.stub_hash = json_data["stub_hash"] + self.port = json_data["port"] + self.board = json_data["board"] + self.variant = json_data["variant"] + # create folder if not self.package_path.exists(): self.package_path.mkdir(parents=True, exist_ok=True) @@ -1030,14 +1038,30 @@ def publish_distribution( log.warning(f"{self.package_name}: Dry run, not saving to database") else: cursor = db_conn.cursor() - variant = "" # TODO: get the variant - row = f""" - INSERT INTO packages (id, name, description, mpy_version, pkg_version, publish, stub_sources, path, hash, stub_hash, port, board, variant) - VALUES ({key}, '{self.package_name}', '{self.description}', '{self.mpy_version}', - '{self.pkg_version}', {self._publish}, '{json.dumps(self.stub_sources)}', '{self.package_path}, '{variant}', - '{self.hash}', '{self.stub_hash} ', '{self.port}', '{self.board}'); - """ - cursor.execute(row) + + d = self.to_dict() + + cursor.execute( + """ + INSERT INTO packages (name, description, mpy_version, pkg_version, publish, stub_sources, path, hash, stub_hash, port, board, variant) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + d["name"], + d["description"], + d["mpy_version"], + d["pkg_version"], + d["publish"], + json.dumps(d["stub_sources"]), + d["path"], + d["hash"], + d["stub_hash"], + d["port"], + d["board"], + d["variant"], + ), + ) + # # get the package state and add it to the database # db_conn.add(self.to_dict()) db_conn.commit() diff --git a/tests/publish/conftest.py b/tests/publish/conftest.py index 68e2ae90..c228cfa4 100644 --- a/tests/publish/conftest.py +++ b/tests/publish/conftest.py @@ -1,17 +1,18 @@ """pytest fixtures for publish tests""" import shutil +import sqlite3 from pathlib import Path import pytest -from pysondb import PysonDB from pytest_mock import MockerFixture - from stubber.publish.package import create_package from .fakeconfig import FakeConfig + pytestmark = [pytest.mark.stubber] + @pytest.fixture def fake_package(request, mocker: MockerFixture, tmp_path: Path, pytestconfig: pytest.Config): """\ @@ -32,26 +33,28 @@ def fake_package(request, mocker: MockerFixture, tmp_path: Path, pytestconfig: p # use default version version = "1.19.1" pkg = create_package("micropython-fake-stubs", mpy_version=version, port="esp32") - pkg._publish = False # type: ignore + pkg._publish = False # type: ignore pkg.create_license() pkg.create_readme() yield pkg @pytest.fixture -def temp_db( +def temp_db_conn( pytestconfig: pytest.Config, tmp_path: Path, ): """""" - db_src = pytestconfig.rootpath / "tests/publish/data/package_data_test.jsondb" - db_path = tmp_path / "package_data_test.jsondb" + db_src = pytestconfig.rootpath / "tests/publish/data/all_packages_test.db" + db_path = tmp_path / "all_packages_test.db" # copy file to temp location shutil.copy(db_src, db_path) - db = PysonDB(db_path.as_posix()) - def fake_commit(): + db_conn = sqlite3.connect(db_path) + db_conn.row_factory = sqlite3.Row # return rows as dicts + yield db_conn + try: + db_conn.close() + db_path.unlink() + except Exception as e: pass - - db.commit = fake_commit - yield db diff --git a/tests/publish/data/all_packages_test.db b/tests/publish/data/all_packages_test.db index c937e44d..8b1807d1 100644 Binary files a/tests/publish/data/all_packages_test.db and b/tests/publish/data/all_packages_test.db differ diff --git a/tests/publish/test_package.py b/tests/publish/test_package.py index 1c1f5b24..d6b2160d 100644 --- a/tests/publish/test_package.py +++ b/tests/publish/test_package.py @@ -5,7 +5,6 @@ import pytest from pytest_mock import MockerFixture - from stubber.publish.enums import StubSource from stubber.publish.package import create_package, package_name from stubber.publish.stubpackage import StubPackage @@ -51,7 +50,9 @@ def test_package_name(family, port, board, expected): ], ) # CORE_STUBS -def test_create_package(tmp_path, pytestconfig, version, port, board, mocker, monkeypatch: pytest.MonkeyPatch): +def test_create_package( + tmp_path, pytestconfig, version, port, board, mocker, monkeypatch: pytest.MonkeyPatch +): """ " test Create a new package with the DOC_STUBS type - test the different methods to manipulate the package on disk @@ -95,6 +96,9 @@ def test_create_package(tmp_path, pytestconfig, version, port, board, mocker, mo read_db_data = [ { "name": "foo-bar-stubs", + "port": "foo", + "board": "GENERIC", + "variant": "", "mpy_version": "1.18", "publish": True, "pkg_version": "1.18.post6", @@ -110,6 +114,9 @@ def test_create_package(tmp_path, pytestconfig, version, port, board, mocker, mo }, { "name": "foo-bar-stubs", + "port": "foo", + "board": "bar", + "variant": "", "mpy_version": "1.18", "publish": True, "pkg_version": "1.18.post6", @@ -125,6 +132,9 @@ def test_create_package(tmp_path, pytestconfig, version, port, board, mocker, mo }, { "name": "foo-bar-stubs", + "port": "foo", + "board": "bar", + "variant": "ota", "mpy_version": "1.18", "publish": True, "pkg_version": "1.18.post6", @@ -163,13 +173,17 @@ def test_package_from_json(tmp_path, pytestconfig, mocker: MockerFixture, json): ) -def run_common_package_tests(package: StubPackage, pkg_name, publish_path: Path, stub_path: Path, test_build=True): +def run_common_package_tests( + package: StubPackage, pkg_name, publish_path: Path, stub_path: Path, test_build=True +): # sourcery skip: no-long-functions "a series of tests to re-use for all packages" assert isinstance(package, StubPackage) assert package.package_name == pkg_name # Package path - assert package.package_path.relative_to(publish_path), "package path should be relative to publish path" + assert package.package_path.relative_to( + publish_path + ), "package path should be relative to publish path" assert (package.package_path).exists() assert (package.package_path / "pyproject.toml").exists() # package path is all lowercase @@ -207,12 +221,16 @@ def run_common_package_tests(package: StubPackage, pkg_name, publish_path: Path, return package.copy_stubs() - filelist = list((package.package_path).rglob("*.py")) + list((package.package_path).rglob("*.pyi")) + filelist = list((package.package_path).rglob("*.py")) + list( + (package.package_path).rglob("*.pyi") + ) assert len(filelist) >= 1 # do it all at once package.update_package_files() - filelist = list((package.package_path).rglob("*.py")) + list((package.package_path).rglob("*.pyi")) + filelist = list((package.package_path).rglob("*.py")) + list( + (package.package_path).rglob("*.pyi") + ) assert len(filelist) >= 1 package.update_pyproject_stubs() @@ -238,5 +256,7 @@ def run_common_package_tests(package: StubPackage, pkg_name, publish_path: Path, assert len(filelist) >= 2 package.clean() - filelist = list((package.package_path).rglob("*.py")) + list((package.package_path).rglob("*.pyi")) + filelist = list((package.package_path).rglob("*.py")) + list( + (package.package_path).rglob("*.pyi") + ) assert len(filelist) == 0 diff --git a/tests/publish/test_publish.py b/tests/publish/test_publish.py index 52655aba..88347652 100644 --- a/tests/publish/test_publish.py +++ b/tests/publish/test_publish.py @@ -1,11 +1,10 @@ """Test publish module - refactored""" +import sqlite3 from pathlib import Path import pytest from mock import MagicMock -from packaging.version import parse -from pysondb import PysonDB from pytest_mock import MockerFixture from stubber.publish.stubpackage import StubPackage @@ -127,10 +126,9 @@ def test_publish_package( tmp_path: Path, pytestconfig: pytest.Config, fake_package: StubPackage, - temp_db: PysonDB, + temp_db_conn: sqlite3.Connection, ): pkg = fake_package - db = temp_db m_publish: MagicMock = mocker.patch( "stubber.publish.package.StubPackage.poetry_publish", autospec=True, return_value=True @@ -140,7 +138,7 @@ def test_publish_package( pkg._publish = True # type: ignore # FIXME : dependency to access to test.pypi.org - result = pkg.publish_distribution_ifchanged(production=False, force=False, db_conn=db) + result = pkg.publish_distribution_ifchanged(production=False, force=False, db_conn=temp_db_conn) assert result, "should be ok" @@ -158,18 +156,21 @@ def test_publish_package( assert m_publish.called, "should call poetry publish" # check is the hashes are added to the database - recs = db.get_by_query( - query=lambda x: x["mpy_version"] == pkg.mpy_version and x["name"] == pkg.package_name + cursor = temp_db_conn.cursor() + cursor.execute( + "SELECT * FROM packages where name = ? AND mpy_version = ? ORDER by pkg_version DESC", + (pkg.package_name, pkg.mpy_version), ) - # dict to list - recs = [{"id": key, "data": recs[key]} for key in recs] - # sort - packages = sorted(recs, key=lambda x: parse(x["data"]["pkg_version"])) - - assert packages[-1]["data"]["name"] == pkg.package_name, "should be the same package name" - assert ( - packages[-1]["data"]["pkg_version"] == pkg.pkg_version - ), "should be the same package version" - assert packages[-1]["data"]["mpy_version"] == pkg.mpy_version, "should be the same mpy version" - assert packages[-1]["data"]["hash"] == pkg.hash, "should be the same hash" - assert packages[-1]["data"]["stub_hash"] == pkg.stub_hash, "should be the same stub hash" + cursor.row_factory = sqlite3.Row # to get dict like access + recs = cursor.fetchall() + + row = recs[0] + assert row["name"] == pkg.package_name, "should be the same package name" + assert row["port"] == pkg.port, "should be the same port" + assert row["board"] == pkg.board, "should be the same board" + assert row["variant"] == pkg.variant, "should be the same variant" + + assert row["pkg_version"] == pkg.pkg_version, "should be the same package version" + assert row["mpy_version"] == pkg.mpy_version, "should be the same mpy version" + assert row["hash"] == pkg.hash, "should be the same hash" + assert row["stub_hash"] == pkg.stub_hash, "should be the same stub hash"