Merge pull request #638 from Josverl/stubber/fix_is_changed

Stubber: Fix retrieving the hashes from the database during build & publish.

src/stubber/publish/package.py

@@ -11,7 +11,6 @@
 from mpflash.logger import log
 from mpflash.versions import clean_version
 
-
 from stubber.publish.defaults import GENERIC, GENERIC_L, default_board
 from stubber.publish.enums import StubSource
 from stubber.publish.stubpackage import StubPackage, StubSources
@@ -50,7 +49,14 @@ def get_package(
         mpy_version=version,
     ):
         # create package from the information retrieved from the database
-        return StubPackage(pkg_name, port, board=board, version=version, json_data=package_info)
+        p_db =  StubPackage(
+            pkg_name,
+            port,
+            board=board,
+            version=version,
+            json_data=package_info,
+        )
+        return p_db
 
     log.debug(f"No package found for {pkg_name} in database, creating new package")
     return create_package(
@@ -71,7 +77,7 @@ def get_package_info(
 ) -> Union[Dict, None]:
     """
     get a package's record from the json db if it can be found
-    matches om the package name and version
+    matches on the package name and version
         pkg_name: package name (micropython-esp32-stubs)
         mpy_version: micropython/firmware version (1.18)
     """
@@ -86,15 +92,14 @@ def get_package_info(
     """,
         (pkg_name, f"{mpy_version}%"),
     )
-    packages = cursor.fetchall()
+    packages = [dict(row) for row in cursor.fetchall()]
 
-    if len(packages) > 0:
-        pkg_from_db = dict(packages[-1])
+    if packages:
+        pkg_from_db = packages[0]
 
         log.debug(f"Found latest {pkg_name} == {pkg_from_db['pkg_version']}")
         return pkg_from_db
-    else:
-        return None
+    return None
 
 
 def create_package(

src/stubber/publish/publish.py

@@ -74,7 +74,7 @@ def publish_multiple(
     worklist = build_worklist(family, versions, ports, boards)
 
     if len(worklist) == 0:
-        log.error("Could not find any packages than can be published.")
+        log.error("Could not find any packages that can be published.")
         return results
 
     for todo in worklist:
@@ -113,7 +113,7 @@ def build_worklist(
     worklist = filter_list(worklist, ports, boards)
 
     for b in boards:
-        if b == "auto":
+        if b in ["auto", "all", "*"]:
             continue
         if not any(i for i in worklist if i["board"].lower() == b.lower()):
             log.warning(f"Could not find any package candidate for board {b}")

src/stubber/publish/stubpackage.py

@@ -82,6 +82,7 @@ def __str__(self) -> str:
         return f"{self.package_name}=={self.mpy_version}"
 
     def __repr__(self) -> str:
+        # todo: use pkg_version to allow for comparing different versions of the same package
         return f"{self.package_name}=={self.mpy_version}"
 
     def __eq__(self, o: object) -> bool:
@@ -388,10 +389,12 @@ def write_package_json(self) -> None:
             json.dump(self.to_dict(), f, indent=4)
 
     def to_dict(self) -> dict:
-        """return the package as a dict to store in the jsondb
+        """return the package as a Dict[str,str] to store in the db
 
         need to simplify some of the Objects to allow serialization to json
-        - the paths to posix paths
+        - stub_sources
+            List[Tuple[StubSource, Path]] -> List[List[str, str]]  # convert
+            the paths to posix paths
         - the version (semver) to a string
         - toml file to list of lines
 
@@ -402,7 +405,9 @@ def to_dict(self) -> dict:
             "publish": self._publish,
             "pkg_version": str(self.pkg_version),
             "path": self.package_path.name,  # only store the folder name , as it is relative to the publish folder
-            "stub_sources": [(name, Path(path).as_posix()) for (name, path) in self.stub_sources],
+            "stub_sources": json.dumps(
+                [(name, Path(path).as_posix()) for (name, path) in self.stub_sources]
+            ),
             "description": self.description,
             "hash": self.hash,
             "stub_hash": self.stub_hash,
@@ -432,7 +437,7 @@ def from_dict(self, json_data: Dict) -> None:
         # set pkg version after creating the toml file
         self.pkg_version = json_data["pkg_version"]
         self.stub_sources = []
-        for name, path in json_data["stub_sources"]:
+        for name, path in json.loads(json_data["stub_sources"]):
             if path.startswith("stubs/"):
                 path = path.replace("stubs/", "")
             self.stub_sources.append((name, Path(path)))
@@ -538,13 +543,13 @@ def create_readme(self) -> None:
         with open(self.package_path / "README.md", "w") as f:
             f.write(f"# {self.package_name}\n\n")
             f.write(TEMPLATE_README)
-            f.write(f"Included stubs:\n")
+            f.write("Included stubs:\n")
             for name, folder in self.stub_sources:
                 f.write(f"* {name} from `stubs/{Path(folder).as_posix()}`\n")
 
-            f.write(f"\n\n")
-            f.write(f"origin | Family | Port | Board | Version\n")
-            f.write(f"-------|--------|------|-------|--------\n")
+            f.write("\n\n")
+            f.write("origin | Family | Port | Board | Version\n")
+            f.write("-------|--------|------|-------|--------\n")
             try:
                 f.write(
                     f"Firmware | {firmware_stubs['firmware']['family']} | {firmware_stubs['firmware']['port']} | {firmware_stubs['firmware']['machine']} | {clean_version(firmware_stubs['firmware']['version'])} \n"
@@ -788,6 +793,14 @@ def __init__(
             STUB_PATH - root-relative path to the folder where the stubs are stored ('./stubs').
 
         """
+        super().__init__(
+            package_name=package_name,
+            mpy_version=clean_version(version, drop_v=True),  # Initial version
+            port=port,
+            board=board,
+            description=description,
+            stubs=stubs or [],
+        )
         self.port = port
         self.board = board
         if json_data is not None:
@@ -796,14 +809,14 @@ def __init__(
             # store essentials
             self.package_name = package_name
             self.description = description
-            self.mpy_version = clean_version(version, drop_v=True)  # Initial version
+            # self.mpy_version = clean_version(version, drop_v=True)  # Initial version
 
             self.create_update_pyproject_toml()
 
-            self.stub_sources: StubSources = []
-            # save the stub sources
-            if stubs:
-                self.stub_sources = stubs
+            # self.stub_sources: StubSources = []
+            # # save the stub sources
+            # if stubs:
+            #     self.stub_sources = stubs
 
         self.status: Status = Status(
             {
@@ -814,14 +827,6 @@ def __init__(
                 "path": self.package_path.as_posix(),
             }
         )
-        super().__init__(
-            package_name=package_name,
-            mpy_version=self.mpy_version,
-            port=port,
-            board=board,
-            description=description,
-            stubs=self.stub_sources,
-        )
 
     def update_sources(self) -> StubSources:
         """
@@ -875,7 +880,7 @@ def update_distribution(self, production: bool) -> bool:
             log.debug(
                 f"{self.package_name}: skipping as one or more source stub folders are missing"
             )
-            self.status["error"] = "Skipped, stub folder(s) missing"
+            self.status["error"] = "Skipped, No stubs found."
             shutil.rmtree(self.package_path.as_posix())
             self._publish = False  # type: ignore
             return False
@@ -1052,7 +1057,7 @@ def publish_distribution(
                     d["mpy_version"],
                     d["pkg_version"],
                     d["publish"],
-                    json.dumps(d["stub_sources"]),
+                    d["stub_sources"],
                     d["path"],
                     d["hash"],
                     d["stub_hash"],

tests/publish/conftest.py

@@ -40,18 +40,18 @@ def fake_package(request, mocker: MockerFixture, tmp_path: Path, pytestconfig: p
 
 
 @pytest.fixture
-def temp_db_conn(
+def test_db_conn(
     pytestconfig: pytest.Config,
     tmp_path: Path,
 ):
     """"""
-    db_src = pytestconfig.rootpath / "tests/publish/data/all_packages_test.db"
+    db_src = pytestconfig.rootpath / "tests/publish/data/test_packages.db"
     db_path = tmp_path / "all_packages_test.db"
     # copy file to temp location
     shutil.copy(db_src, db_path)
 
     db_conn = sqlite3.connect(db_path)
-    db_conn.row_factory = sqlite3.Row # return rows as dicts
+    db_conn.row_factory = sqlite3.Row  # return rows as dicts
     yield db_conn
     try:
         db_conn.close()

tests/publish/test_get_package.py

@@ -0,0 +1,122 @@
+import sqlite3
+import tempfile
+from pathlib import Path
+
+import pytest
+from stubber.publish.package import get_package, get_package_info
+from stubber.publish.stubpackage import StubPackage
+
+
+@pytest.mark.parametrize(
+    "mpy_version, pkg_name",
+    [
+        ("1.18", "micropython-esp32-stubs"),
+        ("1.20.0", "micropython-rp2-pico-stubs"),
+        ("1.22.1", "micropython-rp2-rpi_pico-stubs"),
+        # TODO:  Fix naming of these packages
+        ("1.23.0", "micropython-esp32-esp32_generic_c3-stubs"),
+        ("1.23.0", "micropython-esp32-esp32_generic-stubs"),
+    ],
+)
+# @pytest.mark.parametrize("pkg_name", ["micropython-esp32-stubs"])
+def test_get_package_info(test_db_conn, pkg_name, mpy_version):
+
+    pub_path = Path(tempfile.gettempdir())
+
+    package_info = get_package_info(
+        test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version
+    )
+    assert package_info is not None
+    assert package_info["name"] == pkg_name
+    assert package_info["mpy_version"] == mpy_version
+    assert package_info["pkg_version"].startswith(mpy_version)
+    assert package_info["pkg_version"] > mpy_version
+    # Hashes
+    assert package_info["stub_hash"], "stub only hash not found"
+    assert package_info["hash"], "Hash not found"
+    # Stub sources - string
+    assert package_info["stub_sources"], "Stub sources not found"
+    assert isinstance(package_info["stub_sources"], str), "Stub sources is not a string"
+
+
+def test_get_package_info_no_match(test_db_conn, tmp_path):
+    pub_path = tmp_path
+    pkg_name = "micropython-esp32-stubs"
+    mpy_version = "1.13.1"
+
+    package_info = get_package_info(
+        test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version
+    )
+    assert package_info is None
+
+    def test_get_package_info_invalid_version(test_db_conn):
+        pub_path = Path(tempfile.gettempdir())
+        pkg_name = "micropython-esp32-stubs"
+        mpy_version = "invalid_version"
+
+        with pytest.raises(ValueError):
+            get_package_info(test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version)
+
+
+# def test_get_package_info_empty_package_name(test_db_conn):
+#     pub_path = Path(tempfile.gettempdir())
+#     pkg_name = ""
+#     mpy_version = "1.18"
+
+#     with pytest.raises(ValueError):
+#         get_package_info(test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version)
+
+
+@pytest.mark.parametrize(
+    "mpy_version, port, board",
+    [
+        ("1.18", "esp32", "generic"),
+        ("1.20.0", "rp2", "pico"),
+        ("1.22.1", "rp2", "rpi_pico"),
+        # ("1.23.0", "esp32", "generic_c3"),
+        # ("1.23.0", "esp32", "generic"),
+    ],
+)
+def test_get_package(test_db_conn, mpy_version, port, board):
+    package = get_package(test_db_conn, port=port, board=board, version=mpy_version)
+    assert package is not None
+    assert package.port == port
+    assert package.board == board or board == "generic"
+    assert package.mpy_version == mpy_version
+    assert package.hash, "Hash not found"
+    assert package.stub_hash, "Stub only hash not found"
+
+
+def test_qandd():
+    pkg_name = "micropython-esp32-stubs"
+    port = "esp32"
+    board = "generic"
+    version = "1.18"
+    package_info = {
+        "id": 614587032550805777,
+        "name": "micropython-esp32-stubs",
+        "description": "MicroPython stubs",
+        "mpy_version": "1.18",
+        "pkg_version": "1.18.post3",
+        "publish": 1,
+        "stub_sources": '[["Merged stubs", "micropython-v1_18-esp32-merged"], ["Frozen stubs", "micropython-v1_18-frozen/esp32/GENERIC"], ["Core Stubs", "micropython-core"]]',
+        "path": "micropython-v1_18-esp32-stubs",
+        "hash": "ea79cc2c8b929dd33ac489fcd882b0272d75b83d",
+        "stub_hash": "50ef8cbd42e9599a892217ce1ef16cbed98e2466 ",
+        "port": "esp32",
+        "board": "",
+        "variant": "",
+    }
+
+    p_db = StubPackage(
+        pkg_name,
+        port,
+        board=board,
+        version=version,
+        json_data=package_info,
+    )
+
+    assert p_db is not None
+    assert p_db.port == port
+    assert p_db.hash, "Hash not found"
+    assert p_db.stub_hash, "Stub only hash not found"

tests/publish/test_package.py

@@ -103,11 +103,11 @@ def test_create_package(
         "publish": True,
         "pkg_version": "1.18.post6",
         "path": "foo-v1_18-bar-stubs",
-        "stub_sources": [
+        "stub_sources": """[
             ["MCU stubs", "micropython-v1_17-stm32"],
             ["Frozen stubs", "micropython-v1_17-frozen/stm32/GENERIC"],
-            ["Core Stubs", "cpython_core-pycopy"],
-        ],
+            ["Core Stubs", "cpython_core-pycopy"]
+        ]""",
         "description": "foo bar stubs",
         "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66",
         "stub_hash": "",
@@ -121,11 +121,11 @@ def test_create_package(
         "publish": True,
         "pkg_version": "1.18.post6",
         "path": "foo-v1_18-bar-stubs",
-        "stub_sources": [
+        "stub_sources": """[
             ["MCU stubs", "stubs/micropython-v1_17-stm32"],
             ["Frozen stubs", "stubs/micropython-v1_17-frozen/stm32/GENERIC"],
-            ["Core Stubs", "stubs/cpython_core-pycopy"],
-        ],
+            ["Core Stubs", "stubs/cpython_core-pycopy"]
+        ]""",
         "description": "foo bar stubs",
         "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66",
         "stub_hash": "",
@@ -139,9 +139,9 @@ def test_create_package(
         "publish": True,
         "pkg_version": "1.18.post6",
         "path": "publish/foo-v1_18-bar-stubs",
-        "stub_sources": [
-            ["MCU stubs", "micropython-v1_17-stm32"],
-        ],
+        "stub_sources": """[
+            ["MCU stubs", "micropython-v1_17-stm32"]
+        ]""",
         "description": "foo bar stubs",
         "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66",
         "stub_hash": "1234567890",
@@ -172,7 +172,6 @@ def test_package_from_json(tmp_path, pytestconfig, mocker: MockerFixture, json):
         test_build=False,
     )
 
-
 def run_common_package_tests(
     package: StubPackage, pkg_name, publish_path: Path, stub_path: Path, test_build=True
 ):