Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Patch specifier #13

Merged
merged 3 commits into from
May 3, 2023
Merged

Patch specifier #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e6590f9
correct import for SpecifierSet
seb5g May 3, 2023
bede274
cleaned and updated docs
seb5g May 3, 2023
b1c9bc4
adding docs and info through logger to inform the status of the manag…
seb5g May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions src/pymodaq_plugin_manager/manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,17 @@
from qtpy.QtGui import QTextCursor
from readme_renderer.rst import render

from pymodaq_plugin_manager.validate import validate_json_plugin_list, get_plugins, get_plugin, get_check_repo,\
find_dict_in_list_from_key_val
from pymodaq_plugin_manager.validate import get_plugins
from pymodaq_plugin_manager.validate import get_pypi_pymodaq
from pymodaq_plugin_manager import __version__ as version
from pymodaq_plugin_manager.utils import QVariant, TableModel, TableView, SpinBoxDelegate, get_pymodaq_version

logger = logging.getLogger(__name__)
# logger.addHandler(logging.NullHandler())
logger.addHandler(logging.StreamHandler())


class TableModel(TableModel):

"""Specific Model to display plugins info in a TableView"""
def __init__(self, *args, plugins=[], **kwargs):
super().__init__(*args, **kwargs)
self._selected = [False for ind in range(len(self._data))]
Expand Down Expand Up @@ -72,7 +71,7 @@ def setData(self, index, value, role):


class FilterProxy(QtCore.QSortFilterProxyModel):

"""Utility to filter the View"""
def __init__(self, parent=None):
super().__init__(parent)

Expand Down Expand Up @@ -102,7 +101,10 @@ def setTextFilter(self, regexp):


class PluginManager(QtCore.QObject):
"""Main UI to display a list of plugins and install/uninstall them

Attempts to display plugins only compatible with the currently installed version of pymodaq
"""
quit_signal = Signal()
restart_signal = Signal()

Expand Down Expand Up @@ -329,14 +331,11 @@ def update_model(self, plugin_choice):
self.table_view.setModel(model_proxy)
self.item_clicked(model_proxy.index(0, 0))


def item_clicked(self, index):
if index.isValid():
self.display_info(index)
self.action_button.setEnabled(bool(np.any(index.model().sourceModel().selected)))



def display_info(self, index):
self.info_widget.clear()
if index.isValid():
Expand Down
161 changes: 64 additions & 97 deletions src/pymodaq_plugin_manager/validate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
from hashlib import sha256
from packaging import version
from packaging.version import Version
from packaging.requirements import Requirement, SpecifierSet, Specifier
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
import pkg_resources
from jsonschema import validate
import json
Expand All @@ -19,7 +20,8 @@
import re

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
logger.addHandler(logging.StreamHandler())
logger.setLevel('INFO')

pypi_index = PackageIndex()

Expand All @@ -45,24 +47,32 @@ def find_dict_in_list_from_key_val(dicts, key, value):
return None


def get_pypi_package_list(match_name=None):
def get_pypi_package_list(match_name: str = None) -> List[str]:
"""Connect to the "simple" pypi url to get the list of all packages matching all or part of
the given name

Parameters
----------
match_name: str
The package name to be (partially) matched

Examples
--------
get_pypi_package_list('pymodaq_plugins') will retrieve the names of all packages having 'pymodaq_plugins'
in their name
"""
status = 'Connecting to the pypi repository, may take some time to retrieve the list'
if logging:
logger.info(status)
else:
print(status)
logger.info(status)
simple_package = requests.get('https://pypi.org/simple/')
if simple_package.status_code == 503:
info = 'The service from pypi is currently unavailable, please retry later or install your plugins manually'
if logging:
logger.info(info)
else:
print(info)
logger.info(info)
tree = html.fromstring(simple_package.text)
packages = []
for child in tree.body:
if match_name is None or match_name in child.text:
packages.append(child.text)
logger.info(f'Got package {child.text}')
return packages


Expand All @@ -78,7 +88,21 @@ def get_pymodaq_specifier(requirements: List[str]) -> SpecifierSet:
return specifier


def get_package_metadata(name: str, version: Union[str, Version] = None):
def get_package_metadata(name: str, version: Union[str, Version] = None) -> dict:
"""Retrieve the metadata of a given package on pypi matching or not a specific version

Parameters
----------
name: str
package name
version: Union[str, Version]
package version specifier


Returns
-------
dict of metadata
"""
if version is None:
url = f'https://pypi.python.org/pypi/{name}/json'
else:
Expand All @@ -89,6 +113,7 @@ def get_package_metadata(name: str, version: Union[str, Version] = None):


def get_metadata_from_json(json_as_dict: dict) -> dict:
"""Transform dict of metadata from pypi to a simpler dict"""
try:
if json_as_dict is not None:
metadata = dict([])
Expand Down Expand Up @@ -134,13 +159,24 @@ def get_pypi_pymodaq(package_name='pymodaq-plugins', pymodaq_version: Version =
return get_metadata_from_json(latest)


def get_pypi_plugins(browse_pypi=False, pymodaq_version: Version = None):
def get_pypi_plugins(browse_pypi=True, pymodaq_version: Union[Version, str] = None) -> List[dict]:
"""Fetch the list of plugins (for a given version) of pymodaq

Parameters
----------
browse_pypi: bool
If True get the list from pypi server, if False from the builtin json (deprecated, should be True)
pymodaq_version: Union[str, Version]
a given pymodaq version (or the latest if None)

Returns
-------
list of dictionaries giving info on plugins
"""
plugins = []
if browse_pypi:
packages = get_pypi_package_list('pymodaq-plugins')
else:
packages = [plug['plugin-name'] for plug in get_plugins_from_json()]
packages = get_pypi_package_list('pymodaq-plugins')
for package in packages:
logger.info(f'Fetching metadata for package {package}')
metadata = get_pypi_pymodaq(package, pymodaq_version)
if metadata is not None:
title = metadata['description'].split('\n')[0]
Expand All @@ -158,6 +194,7 @@ def get_pypi_plugins(browse_pypi=False, pymodaq_version: Version = None):


def get_plugin_sourcefile_id(filename):
"""Get the SHA identifier of a vien file"""
h = sha256()
b = bytearray(128*1024)
mv = memoryview(b)
Expand All @@ -167,40 +204,26 @@ def get_plugin_sourcefile_id(filename):
return h.hexdigest()


def get_plugins_from_json():
return validate_json_plugin_list()['pymodaq-plugins']


def get_plugin(name, entry='display-name'):
plugins = get_plugins_from_json()
d = find_dict_in_list_from_key_val(plugins, entry, name)
return d


def get_check_repo(plugin_dict):
"""Unused"""
try:
response = requests.get(plugin_dict["repository"])
except requests.exceptions.RequestException as e:
if logging:
logger.exception(str(e))
return str(e)
logger.exception(str(e))
return str(e)

if response.status_code != 200:
rep = f'{plugin_dict["display-name"]}: failed to download plugin. Returned code {response.status_code}'
if logging:
logger.error(rep)
return rep
logger.error(rep)

# Hash it and make sure its what is expected
hash = sha256(response.content).hexdigest()
if plugin_dict["id"].lower() != hash.lower():
rep = f'{plugin_dict["display-name"]}: Invalid hash. Got {hash.lower()} but expected {plugin_dict["id"]}'
if logging:
logger.error(rep)
logger.error(rep)
return rep
else:
if logging:
logger.info(f'SHA256 is Ok')
logger.info(f'SHA256 is Ok')


def get_plugins(from_json=False, browse_pypi=True, pymodaq_version: Version = None):
Expand All @@ -222,10 +245,7 @@ def get_plugins(from_json=False, browse_pypi=True, pymodaq_version: Version = No
plugins_update: list of plugins with existing update
"""
logger.info('Fetching plugin list')
if from_json:
plugins_available = get_plugins_from_json()
else:
plugins_available = get_pypi_plugins(browse_pypi=browse_pypi, pymodaq_version=pymodaq_version)
plugins_available = get_pypi_plugins(browse_pypi=browse_pypi, pymodaq_version=pymodaq_version)

plugins = deepcopy(plugins_available)
plugins_installed_init = [{'plugin-name': entry.module_name,
Expand All @@ -247,62 +267,8 @@ def get_plugins(from_json=False, browse_pypi=True, pymodaq_version: Version = No
return plugins_available, plugins_installed, plugins_update


def validate_json_plugin_list():
base_path = Path(__file__).parent.joinpath('data')
with open(str(base_path.joinpath('plugin_list.schema'))) as f:
schema = json.load(f)
with open(str(base_path.joinpath('PluginList.json'))) as f:
plugins = json.load(f)
validate(instance=plugins, schema=schema)
return plugins


def post_error(message):
if logging:
logger.error(message)


def check_plugin_entries():
displaynames = []
repositories = []
for plugin in get_plugins_from_json():
if logging:
logger.info(f'Checking info on plugin: {plugin["display-name"]}')

try:
response = requests.get(plugin["repository"])
except requests.exceptions.RequestException as e:
post_error(str(e))
continue

if response.status_code != 200:
post_error(f'{plugin["display-name"]}: failed to download plugin. Returned code {response.status_code}')
continue

# Hash it and make sure its what is expected
hash = sha256(response.content).hexdigest()
if plugin["id"].lower() != hash.lower():
post_error(f'{plugin["display-name"]}: Invalid hash. Got {hash.lower()} but expected {plugin["id"]}')
else:
if logging:
logger.info(f'SHA256 is Ok')

# check uniqueness of json display-name and repository
found = False
for name in displaynames:
if plugin["display-name"] == name:
post_error(f'{plugin["display-name"]}: non unique display-name entry')
found = True
if not found:
displaynames.append(plugin["display-name"])

found = False
for repo in repositories:
if plugin["repository"] == repo:
post_error(f'{plugin["repository"]}: non unique repository entry')
found = True
if not found:
repositories.append(plugin["repository"])
logger.error(message)


def extract_authors_from_description(description):
Expand All @@ -325,6 +291,7 @@ def extract_authors_from_description(description):


def write_plugin_doc():
"""Update the README from info of all available plugins"""
plugins = get_pypi_plugins(browse_pypi=True)
base_path = Path(__file__).parent

Expand Down Expand Up @@ -403,8 +370,8 @@ def write_plugin_doc():
content += writer.dumps()
f.write(content)


if __name__ == '__main__':
#check_plugin_entries()
write_plugin_doc()
# versions = get_pypi_pymodaq()
# from pymodaq_plugin_manager import __version__ as version
Expand Down