diff --git a/.travis.yml b/.travis.yml index 52c5a9a..629bc5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ sudo: false language: python python: -- 2.7 -- 3.6 - 3.7 - 3.8 +- 3.9 install: - pip install -U setuptools - pip install -r requirements.txt #fix versions diff --git a/docs/source/conf.py b/docs/source/conf.py index 3e5973a..21f726b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # skosprovider_getty documentation build configuration file, created by # sphinx-quickstart on Fri Sep 19 10:30:09 2014. @@ -49,8 +48,8 @@ master_doc = 'index' # General information about the project. -project = u'skosprovider_getty' -copyright = u'2014-2020, Flanders Heritage Agency' +project = 'skosprovider_getty' +copyright = '2014-2020, Flanders Heritage Agency' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -202,8 +201,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'skosprovider_getty.tex', u'skosprovider\\_getty Documentation', - u'Flanders Heritage Agency', 'manual'), + ('index', 'skosprovider_getty.tex', 'skosprovider\\_getty Documentation', + 'Flanders Heritage Agency', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -232,8 +231,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'skosprovider_getty', u'skosprovider_getty Documentation', - [u'Flanders Heritage Agency'], 1) + ('index', 'skosprovider_getty', 'skosprovider_getty Documentation', + ['Flanders Heritage Agency'], 1) ] # If true, show URL addresses after external links. @@ -246,8 +245,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'skosprovider_getty', u'skosprovider_getty Documentation', - u'Flanders Heritage Agency', 'skosprovider_getty', 'One line description of project.', + ('index', 'skosprovider_getty', 'skosprovider_getty Documentation', + 'Flanders Heritage Agency', 'skosprovider_getty', 'One line description of project.', 'Miscellaneous'), ] diff --git a/examples/churches.py b/examples/churches.py index 250ae8a..b7b298e 100755 --- a/examples/churches.py +++ b/examples/churches.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- ''' This script demonstrates using the AATProvider to get the concept of Churches. @@ -9,20 +8,20 @@ aat = AATProvider(metadata={'id': 'AAT'}) -churches = aat.get_by_id(300007466) +churches = aat.get_by_id('300007466') -lang = ['en', 'nl', 'es', 'de', 'fr'] +langs = ['en', 'nl', 'es', 'de', 'fr'] print('Label per language') print('------------------') -for l in lang: - label = churches.label(l) - print(l + ' --> ' + label.language + ': ' + label.label + ' [' + label.type + ']') +for lang in langs: + label = churches.label(lang) + print(lang + ' --> ' + label.language + ': ' + label.label + ' [' + label.type + ']') print('All Labels') print('----------') -for l in churches.labels: - print(l.language + ': ' + l.label + ' [' + l.type + ']') +for lang in churches.labels: + print(lang.language + ': ' + lang.label + ' [' + lang.type + ']') print('All Notes') print('---------') diff --git a/examples/collect_subclasses.py b/examples/collect_subclasses.py index 64246d5..9e09415 100644 --- a/examples/collect_subclasses.py +++ b/examples/collect_subclasses.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- ''' This script demonstrates retrieving subclasses for a certain class. ''' diff --git a/examples/expand.py b/examples/expand.py index cf241a1..a96c836 100755 --- a/examples/expand.py +++ b/examples/expand.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- ''' This script demonstrates using the AATProvider to expand a collection ''' @@ -11,4 +10,4 @@ print('Results') print('------') for result in results: - print(result) \ No newline at end of file + print(result) diff --git a/examples/find.py b/examples/find.py index 761df18..53f1850 100755 --- a/examples/find.py +++ b/examples/find.py @@ -1,15 +1,20 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- ''' -This script demonstrates using the AATProvider to find the concepts that are a member of a collection with 'church' in their label +This script demonstrates using the AATProvider to find the concepts that are a member of +a collection with 'church' in their label ''' from skosprovider_getty.providers import AATProvider -results = AATProvider({'id': 'AAT', 'default_language': 'nl'}).find({'label': 'church', 'type': 'concept', 'collection': {'id': '300007466', 'depth': 'all'}}) +results = AATProvider({'id': 'AAT', 'default_language': 'nl'}).find( + { + 'label': 'church', + 'type': 'concept', + 'collection': {'id': '300007466', 'depth': 'all'} + } +) print('Results') print('------') for result in results: print(result) - diff --git a/examples/flanders.py b/examples/flanders.py index 8afc949..d14ddcc 100755 --- a/examples/flanders.py +++ b/examples/flanders.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- ''' This script demonstrates using the TGNProvider to get the concept of Flanders. @@ -11,18 +10,18 @@ flanders = aat.get_by_id(7018236) -lang = ['en', 'nl', 'es', 'de', 'fr'] +langs = ['en', 'nl', 'es', 'de', 'fr'] print('Label per language') print('------------------') -for l in lang: - label = flanders.label(l) - print(l + ' --> ' + label.language + ': ' + label.label + ' [' + label.type + ']') +for lang in langs: + label = flanders.label(lang) + print(lang + ' --> ' + label.language + ': ' + label.label + ' [' + label.type + ']') print('Labels') print('------') -for l in flanders.labels: - print(l.language + ': ' + l.label + ' [' + l.type + ']') +for lang in flanders.labels: + print(lang.language + ': ' + lang.label + ' [' + lang.type + ']') print('Notes') print('-----') diff --git a/requirements-dev.txt b/requirements-dev.txt index 9ac277d..adab7ba 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,8 +4,8 @@ # Testing pytest==4.6.9 ; python_version < '3.0' pytest==5.3.5 ; python_version >= '3.0' -pytest-cov==2.8.1 -coveralls==1.11.1 +pytest-cov==3.0.0 +coveralls==3.3.1 # Documentation Sphinx==1.8.5 ; python_version < '3.5' diff --git a/requirements.txt b/requirements.txt index c9538a7..279fdba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.24.0 -skosprovider==0.7.1 +requests==2.26.0 +skosprovider==1.0.0 #-e git+https://github.com/koenedaele/skosprovider.git@DEV_0.7.0#egg=skosprovider -rdflib==5.0.0 +rdflib==6.0.2 diff --git a/setup.py b/setup.py index 02ecfe4..5c35b4d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import os try: - from setuptools import setup, find_packages + from setuptools import setup except ImportError: from distutils.core import setup diff --git a/skosprovider_getty/providers.py b/skosprovider_getty/providers.py index 45ff0b0..b5e3163 100644 --- a/skosprovider_getty/providers.py +++ b/skosprovider_getty/providers.py @@ -1,31 +1,29 @@ -# -*- coding: utf-8 -*- ''' This module contains classes that implement :class:`skosprovider.providers.VocabularyProvider` against the LOD version of the Getty Vocabularies (AAT, TGN and ULAN). .. note:: - | At initialisation, the Getty providers will search which gvp-classes of the gvp-ontology are a subclass of skos-classes. + | At initialisation, the Getty providers will search which gvp-classes of the + gvp-ontology are a subclass of skos-classes. | This can cause a time delay of several seconds at startup. ''' -import requests -import warnings import logging +import warnings +import requests from language_tags import tags -from requests.exceptions import ConnectionError, Timeout -from skosprovider.exceptions import ProviderUnavailableException from skosprovider.providers import VocabularyProvider -from skosprovider_getty.utils import ( - uri_to_id, uri_to_graph, - conceptscheme_from_uri, - things_from_graph, - SubClassCollector, - GVP, - do_get_request -) + +from skosprovider_getty.utils import GVP +from skosprovider_getty.utils import SubClassCollector +from skosprovider_getty.utils import conceptscheme_from_uri +from skosprovider_getty.utils import do_get_request +from skosprovider_getty.utils import things_from_graph +from skosprovider_getty.utils import uri_to_graph +from skosprovider_getty.utils import uri_to_id log = logging.getLogger(__name__) @@ -49,7 +47,7 @@ def __init__(self, metadata, **kwargs): * The :class:`skosprovider_getty.providers.AATProvider` is the default :class:`skosprovider_getty.providers.GettyProvider` """ - if not 'default_language' in metadata: + if 'default_language' not in metadata: metadata['default_language'] = 'en' if 'subject' not in metadata: metadata['subject'] = [] @@ -91,9 +89,9 @@ def get_by_id(self, id, change_notes=False): :return: corresponding :class:`skosprovider.skos.Concept` or :class:`skosprovider.skos.Concept`. Returns None if non-existing id """ - graph = uri_to_graph('%s/%s.rdf' % (self.url, id), session=self.session) + graph = uri_to_graph(f'{self.url}/{id}.rdf', session=self.session) if graph is False: - log.debug('Failed to retrieve data for %s/%s.rdf' % (self.url, id)) + log.debug(f'Failed to retrieve data for {self.url}/{id}.rdf') return False # get the concept things = things_from_graph( @@ -107,7 +105,6 @@ def get_by_id(self, id, change_notes=False): c = things[0] return c - def get_by_uri(self, uri, change_notes=False): """ Get a :class:`skosprovider.skos.Concept` or :class:`skosprovider.skos.Collection` by uri @@ -187,12 +184,12 @@ def find(self, query, **kwargs): type_c = query['type'] if type_c not in ('all', 'concept', 'collection'): raise ValueError("type: only the following values are allowed: 'all', 'concept', 'collection'") - #Collection to search in (optional) + # Collection to search in (optional) coll_id = None coll_depth = None if 'collection' in query: coll = query['collection'] - if not 'id' in coll: + if 'id' not in coll: raise ValueError("collection: 'id' is required key if a collection-dictionary is given") coll_id = coll['id'] coll_depth = 'members' @@ -201,7 +198,7 @@ def find(self, query, **kwargs): if coll_depth not in ('members', 'all'): raise ValueError( "collection - 'depth': only the following values are allowed: 'members', 'all'") - #Matches (optional) + # Matches (optional) match_uri = None match_pred = 'skos:mappingRelation' if 'matches' in query: @@ -214,7 +211,7 @@ def find(self, query, **kwargs): if match_type: match_pred = 'skos:%sMatch' % match_type - #build sparql query + # build sparql query coll_x = "" if coll_id is not None and coll_depth == 'all': coll_x = "gvp:broaderExtended " + self.vocab_id + ":" + coll_id + ";" @@ -223,7 +220,7 @@ def find(self, query, **kwargs): match_values = "" if match_uri is not None: - match_values = "%s <%s>;" % (match_pred, match_uri) + match_values = f"{match_pred} <{match_uri}>;" type_values = "((?Type = skos:Concept) || (?Type = skos:Collection))" if type_c == 'concept': @@ -231,17 +228,17 @@ def find(self, query, **kwargs): elif type_c == 'collection': type_values = "(?Type = skos:Collection)" query = """ - SELECT ?Subject ?Term ?Type ?Id (lang(?Term) as ?Lang) { - ?Subject rdf:type ?Type; dc:identifier ?Id; skos:inScheme %s:; %s%s%s. - OPTIONAL { - {?Subject xl:prefLabel [skosxl:literalForm ?Term]} - } - FILTER(%s) - }""" % ( + SELECT ?Subject ?Term ?Type ?Id (lang(?Term) as ?Lang) {{ + ?Subject rdf:type ?Type; dc:identifier ?Id; skos:inScheme {}:; {}{}{}. + OPTIONAL {{ + {{?Subject xl:prefLabel [skosxl:literalForm ?Term]}} + }} + FILTER({}) + }}""".format( self.vocab_id, self._build_keywords(label), coll_x, match_values, type_values) - ret= self._get_answer(query, **kwargs) + ret = self._get_answer(query, **kwargs) language = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) @@ -281,11 +278,11 @@ def _get_answer(self, query, **kwargs): else: label = "" item = { - 'id': result["Id"]["value"], - 'uri': uri, - 'type': result["Type"]["value"].rsplit('#', 1)[1].lower(), - 'label': label, - 'lang': result["Lang"]["value"] + 'id': result["Id"]["value"], + 'uri': uri, + 'type': result["Type"]["value"].rsplit('#', 1)[1].lower(), + 'label': label, + 'lang': result["Lang"]["value"] } if uri not in d: @@ -294,13 +291,15 @@ def _get_answer(self, query, **kwargs): pass elif tags.tag(item['lang']).format == tags.tag(self._get_language(**kwargs)).format: d[uri] = item - elif tags.tag(item['lang']).language and (tags.tag(item['lang']).language.format == tags.tag(self._get_language()).language.format): + elif ( + tags.tag(item['lang']).language and ( + tags.tag(item['lang']).language.format == tags.tag(self._get_language()).language.format) + ): d[uri] = item elif tags.tag(item['lang']).format == tags.tag('en').format: d[uri] = item return list(d.values()) - def _get_top(self, type='All', **kwargs): """ Returns all top-level facets. The returned values depend on the given type: Concept or All (Concepts and Collections). Default All is used. @@ -309,21 +308,21 @@ def _get_top(self, type='All', **kwargs): :return: A :class:`lst` of concepts (and collections). """ - if type == "concepts" : + if type == "concepts": type_values = "(?Type = skos:Concept)" else: type_values = "((?Type = skos:Concept) || (?Type = skos:Collection))" query = """SELECT ?Subject ?Id ?Type ?Term (lang(?Term) as ?Lang) - { + {{ ?Subject a gvp:Facet; rdf:type ?Type; - dc:identifier ?Id; skos:inScheme %s:;. - OPTIONAL { - {?Subject xl:prefLabel [skosxl:literalForm ?Term]} - } - FILTER (%s) - }""" % (self.vocab_id, type_values) - ret= self._get_answer(query, **kwargs) + dc:identifier ?Id; skos:inScheme {}:;. + OPTIONAL {{ + {{?Subject xl:prefLabel [skosxl:literalForm ?Term]}} + }} + FILTER ({}) + }}""".format(self.vocab_id, type_values) + ret = self._get_answer(query, **kwargs) language = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) @@ -336,7 +335,6 @@ def get_top_concepts(self, **kwargs): """ return self._get_top("concepts", **kwargs) - def get_top_display(self, **kwargs): """ Returns all concepts or collections that form the top-level of a display hierarchy. @@ -354,16 +352,16 @@ def get_children_display(self, id, **kwargs): type_values = "((?Type = skos:Concept) || (?Type = skos:Collection))" query = """SELECT ?Subject ?Id ?Type ?Term (lang(?Term) as ?Lang) - { + {{ ?Subject rdf:type ?Type; - dc:identifier ?Id; skos:inScheme %s:; gvp:%s %s:%s;. - OPTIONAL { - {?Subject xl:prefLabel [skosxl:literalForm ?Term]} - } - FILTER(%s) - }""" % (self.vocab_id, broader, self.vocab_id, id, type_values) - - ret= self._get_answer(query, **kwargs) + dc:identifier ?Id; skos:inScheme {}:; gvp:{} {}:{};. + OPTIONAL {{ + {{?Subject xl:prefLabel [skosxl:literalForm ?Term]}} + }} + FILTER({}) + }}""".format(self.vocab_id, broader, self.vocab_id, id, type_values) + + ret = self._get_answer(query, **kwargs) language = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) @@ -378,19 +376,19 @@ def expand(self, id): :returns: A :class:`lst` of id's. Returns false if the input id does not exists """ - query = """SELECT DISTINCT ?Id{ - { - ?Subject dc:identifier ?Id; skos:inScheme %s:; gvp:broaderExtended %s;. - } + query = """SELECT DISTINCT ?Id{{ + {{ + ?Subject dc:identifier ?Id; skos:inScheme {}:; gvp:broaderExtended {};. + }} UNION - { - VALUES ?Id {'%s'} - ?Subject dc:identifier ?Id; skos:inScheme %s:; rdf:type skos:Concept. - } - } - """ % (self.vocab_id, self.vocab_id + ":" + id, id, self.vocab_id) - - print (query) + {{ + VALUES ?Id {{'{}'}} + ?Subject dc:identifier ?Id; skos:inScheme {}:; rdf:type skos:Concept. + }} + }} + """.format(self.vocab_id, self.vocab_id + ":" + id, id, self.vocab_id) + + print(query) request = self.base_url + "sparql.json" res = do_get_request(request, self.session, params={'query': query}) r = res.json() @@ -417,7 +415,7 @@ def _sort(self, items, sort, language='en', reverse=False): if sort is None: sort = 'id' if sort == 'sortlabel': - sort='label' + sort = 'label' items.sort(key=lambda item: item[sort], reverse=reverse) return items diff --git a/skosprovider_getty/utils.py b/skosprovider_getty/utils.py index 5d39d3f..2ea7bc3 100644 --- a/skosprovider_getty/utils.py +++ b/skosprovider_getty/utils.py @@ -1,31 +1,33 @@ -# -*- coding: utf-8 -*- - ''' This module contains utility functions for :mod:`skosprovider_getty`. ''' -import requests +import logging + import rdflib +import requests from rdflib.graph import Graph +from rdflib.namespace import DC +from rdflib.namespace import RDF +from rdflib.namespace import RDFS +from rdflib.namespace import SKOS from rdflib.term import URIRef -from requests.exceptions import ConnectionError, Timeout +from requests.exceptions import ConnectionError +from requests.exceptions import Timeout from skosprovider.exceptions import ProviderUnavailableException +from skosprovider.skos import Collection +from skosprovider.skos import Concept +from skosprovider.skos import ConceptScheme +from skosprovider.skos import Label +from skosprovider.skos import Note -from skosprovider.skos import ( - Concept, - Collection, - Label, - Note, - ConceptScheme) - -import logging log = logging.getLogger(__name__) -from rdflib.namespace import RDFS, RDF, SKOS, DC PROV = rdflib.Namespace('http://www.w3.org/ns/prov#') ISO = rdflib.Namespace('http://purl.org/iso25964/skos-thes#') GVP = rdflib.Namespace('http://vocab.getty.edu/ontology#') + def conceptscheme_from_uri(conceptscheme_uri, **kwargs): ''' Read a SKOS Conceptscheme from a :term:`URI` @@ -75,14 +77,14 @@ def things_from_graph(graph, subclasses, conceptscheme, **kwargs): con = Concept( uri_to_id(uri), uri=uri, - concept_scheme = conceptscheme, - labels = _create_from_subject_typelist(graph, sub, valid_label_types), - notes = _create_from_subject_typelist(graph, sub, hierarchy_notetypes(Note.valid_types)), - sources = [], - broader = _create_from_subject_predicate(graph, sub, SKOS.broader), - narrower = _create_from_subject_predicate(graph, sub, SKOS.narrower), - related = _create_from_subject_predicate(graph, sub, SKOS.related), - subordinate_arrays = _create_from_subject_predicate(graph, sub, ISO.subordinateArray), + concept_scheme=conceptscheme, + labels=_create_from_subject_typelist(graph, sub, valid_label_types), + notes=_create_from_subject_typelist(graph, sub, hierarchy_notetypes(Note.valid_types)), + sources=[], + broader=_create_from_subject_predicate(graph, sub, SKOS.broader), + narrower=_create_from_subject_predicate(graph, sub, SKOS.narrower), + related=_create_from_subject_predicate(graph, sub, SKOS.related), + subordinate_arrays=_create_from_subject_predicate(graph, sub, ISO.subordinateArray), matches=matches ) clist.append(con) @@ -92,12 +94,12 @@ def things_from_graph(graph, subclasses, conceptscheme, **kwargs): col = Collection( uri_to_id(uri), uri=uri, - concept_scheme = conceptscheme, - labels = _create_from_subject_typelist(graph, sub, valid_label_types), - notes = _create_from_subject_typelist(graph, sub, hierarchy_notetypes(Note.valid_types)), - sources = [], - members = _create_from_subject_predicate(graph, sub, SKOS.member), - superordinates = _get_super_ordinates(conceptscheme, sub, session=s) + concept_scheme=conceptscheme, + labels=_create_from_subject_typelist(graph, sub, valid_label_types), + notes=_create_from_subject_typelist(graph, sub, hierarchy_notetypes(Note.valid_types)), + sources=[], + members=_create_from_subject_predicate(graph, sub, SKOS.member), + superordinates=_get_super_ordinates(conceptscheme, sub, session=s) ) clist.append(col) @@ -108,7 +110,7 @@ def _create_from_subject_typelist(graph, subject, typelist): list = [] note_uris = [] for p in typelist: - term = SKOS.term(p) + term = SKOS._NS.term(p) list.extend(_create_from_subject_predicate(graph, subject, term, note_uris)) return list @@ -116,8 +118,8 @@ def _create_from_subject_typelist(graph, subject, typelist): def _get_super_ordinates(conceptscheme, sub, **kwargs): ret = [] s = kwargs.get('session', requests.Session()) - query = """PREFIX ns:<%s> - SELECT * WHERE {?s iso-thes:subordinateArray ns:%s}""" % (conceptscheme.uri, uri_to_id(sub)) + query = """PREFIX ns:<{}> + SELECT * WHERE {{?s iso-thes:subordinateArray ns:{}}}""".format(conceptscheme.uri, uri_to_id(sub)) url = conceptscheme.uri.strip('/').rsplit('/', 1)[0] + "/sparql.json" res = do_get_request(url, s, params={'query': query}) r = res.json() @@ -150,18 +152,18 @@ def _create_label(literal, type): if language is None: language = 'und' try: - l = Label(literal.toPython(), type, language) - except ValueError as e: + lang = Label(literal.toPython(), type, language) + except ValueError: log.warn('Received a label with an invalid language tag: %s.', language) - l = Label(literal.toPython(), type, 'und') - return l + lang = Label(literal.toPython(), type, 'und') + return lang def _create_note(graph, uri, type, change_notes=False): if not change_notes and '/rev/' in uri: return None else: - note = u'' + note = '' language = 'en' # http://vocab.getty.edu/aat/scopeNote @@ -184,6 +186,7 @@ class SubClassCollector: ''' A utility class to collect all the subclasses of a certain Class from an ontology file. ''' + def __init__(self, namespace): self.ontology_graphs = {} self.namespace = namespace @@ -232,12 +235,12 @@ def collect_subclasses(self, clazz): if self.namespace not in self.ontology_graphs: try: graph = rdflib.Graph() - result = graph.parse(str(self.namespace), format="application/rdf+xml") + graph.parse(str(self.namespace), format="application/rdf+xml") self.ontology_graphs[self.namespace] = graph - except: # pragma: no cover + except: # pragma: no cover # noqa: E722 self.ontology_graphs[self.namespace] = None g = self.ontology_graphs[self.namespace] - if not g is None: + if g is not None: for sub, pred, obj in g.triples((None, RDFS.subClassOf, None)): self._is_subclass_of(sub, clazz) return self.subclasses[clazz] @@ -249,12 +252,12 @@ def _is_subclass_of(self, subject, clazz): if namespace not in self.ontology_graphs: try: graph = rdflib.Graph() - result = graph.parse(str(namespace), format="application/rdf+xml") + graph.parse(str(namespace), format="application/rdf+xml") self.ontology_graphs[namespace] = graph - except: # pragma: no cover + except: # pragma: no cover # noqa: E722 self.ontology_graphs[namespace] = None g = self.ontology_graphs[namespace] - if not g is None: + if g is not None: for sub, pred, obj in g.triples((subject, RDFS.subClassOf, None)): if obj in self.subclasses[clazz]: self.subclasses[clazz].append(subject) @@ -297,7 +300,8 @@ def uri_to_graph(uri, **kwargs): res = do_get_request(uri, s) if res.status_code == 404: return False - graph.parse(data=res.content) + content_type = res.headers['Content-Type'].split(';')[0] + graph.parse(data=res.content, format=content_type) return graph @@ -307,11 +311,12 @@ def do_get_request(url, session=None, headers=None, params=None): try: res = session.get(url, headers=headers, params=params) except ConnectionError: - raise ProviderUnavailableException("Request could not be executed due to connection issues- Request: %s" % (url,)) - except Timeout: # pragma: no cover - raise ProviderUnavailableException("Request could not be executed due to timeout - Request: %s" % (url,)) + raise ProviderUnavailableException(f"Request could not be executed due to connection issues- Request: {url}") + except Timeout: # pragma: no cover + raise ProviderUnavailableException(f"Request could not be executed due to timeout - Request: {url}") if res.status_code >= 500: - raise ProviderUnavailableException("Request could not be executed due to server issues - Request: %s. Response: %s." % (url, res.content)) + raise ProviderUnavailableException( + f"Request could not be executed due to server issues - Request: {url}. Response: {res.content}.") if not res.encoding: res.encoding = 'utf-8' return res diff --git a/tests/test_providers.py b/tests/test_providers.py index e3bdb44..4a0fb7c 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,18 +1,14 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- -import pytest -from rdflib.namespace import SKOS, Namespace +import unittest +import pytest +import requests from skosprovider.exceptions import ProviderUnavailableException -from skosprovider_getty.providers import ( - AATProvider, - TGNProvider, - ULANProvider, - GettyProvider -) -import unittest -from skosprovider_getty.utils import SubClassCollector +from skosprovider_getty.providers import AATProvider +from skosprovider_getty.providers import GettyProvider +from skosprovider_getty.providers import TGNProvider +from skosprovider_getty.providers import ULANProvider global clazzes, ontologies clazzes = [] @@ -22,7 +18,6 @@ class GettyProviderTests(unittest.TestCase): def test_set_custom_session(self): - import requests sess = requests.Session() provider = AATProvider({'id': 'AAT'}, session=sess) self.assertEqual(sess, provider.session) @@ -36,7 +31,7 @@ def test_allowed_instance_scopes(self): def test_override_instance_scopes(self): provider = AATProvider( {'id': 'AAT'}, - allowed_instance_scopes = ['single'] + allowed_instance_scopes=['single'] ) assert provider.allowed_instance_scopes == ['single'] @@ -69,14 +64,14 @@ def test_get_by_id_concept(self): self.assertEqual(concept['type'], 'concept') self.assertIsInstance(concept['labels'], list) - preflabels = [{'nl': 'kerken'}, {'de': u'Kirche (Gebäude)'}] + preflabels = [{'nl': 'kerken'}, {'de': 'Kirche (Gebäude)'}] preflabels_conc = [{label.language: label.label} for label in concept['labels'] if label.type == 'prefLabel'] self.assertGreater(len(preflabels_conc), 0) for label in preflabels: self.assertIn(label, preflabels_conc) - altlabels = [{u'nl': 'kerk'}, {u'de': u'kirchen (Gebäude)'}] + altlabels = [{'nl': 'kerk'}, {'de': 'kirchen (Gebäude)'}] altlabels_conc = [{label.language: label.label} for label in concept['labels'] if label.type == 'altLabel'] self.assertGreater(len(altlabels_conc), 0) @@ -86,8 +81,10 @@ def test_get_by_id_concept(self): self.assertGreater(len(concept['notes']), 0) self.assertEqual(concept['id'], '300007466') - #todo gvp:broader is not a subproperty of skos:broader anymore. This is the reason why there are no broader elements anymore belonging to the Concept...to be decided what to do... - #self.assertEqual(concept['broader'][0], '300007391') + # todo gvp:broader is not a subproperty of skos:broader anymore. This is the + # reason why there are no broader elements anymore belonging to the Concept... + # to be decided what to do... + # self.assertEqual(concept['broader'][0], '300007391') self.assertIn('300312247', concept['related']) def test_get_by_id_collection(self): @@ -95,8 +92,9 @@ def test_get_by_id_collection(self): assert collection is not False assert collection.uri == 'http://vocab.getty.edu/aat/300007473' assert collection.type == 'collection' - assert u'' in [label.label for label in collection.labels if label.language == 'nl' and label.type == 'prefLabel'] - assert len(collection.notes) == 0 + assert '' in [ + label.label for label in collection.labels if label.language == 'nl' and label.type == 'prefLabel'] + assert len(collection.notes) == 0 def test_get_by_id_invalid(self): concept = AATProvider({'id': 'AAT'}).get_by_id('123') @@ -114,7 +112,7 @@ def test_get_by_id_subOrdinateArrays(self): concept = concept.__dict__ self.assertEqual(concept['id'], '300138225') self.assertIn('300138225-array', concept['subordinate_arrays']) - #300126352 + # 300126352 def test_get_by_uri(self): # Default GettyProvider is an AAT provider @@ -134,7 +132,7 @@ def test_get_by_id_tgn(self): concept = TGNProvider({'id': 'TGN'}).get_by_id('1000063') concept = concept.__dict__ self.assertEqual(concept['uri'], 'http://vocab.getty.edu/tgn/1000063') - self.assertIn(u'België', [label.label for label in concept['labels'] + self.assertIn('België', [label.label for label in concept['labels'] if label.language == 'nl' and label.type == 'prefLabel']) def test_get_all(self): @@ -149,11 +147,11 @@ def test_get_top_display(self): keys_first_display = top_TGN_display[0].keys() for key in ['id', 'type', 'label', 'uri']: self.assertIn(key, keys_first_display) - self.assertIn(u'World', [label['label'] for label in top_TGN_display]) + self.assertIn('World', [label['label'] for label in top_TGN_display]) top_AAT_display = AATProvider({'id': 'AAT', 'default_language': 'nl'}).get_top_display() self.assertIsInstance(top_AAT_display, list) self.assertGreater(len(top_AAT_display), 0) - self.assertIn(u'Facet Stijlen en perioden', [label['label'] for label in top_AAT_display]) + self.assertIn('Facet Stijlen en perioden', [label['label'] for label in top_AAT_display]) def test_get_top_concepts(self): kwargs = {'language': 'nl'} @@ -163,13 +161,14 @@ def test_get_top_concepts(self): def test_get_childeren_display(self): kwargs = {'language': 'nl'} - childeren_tgn_belgie = TGNProvider({'id': 'TGN', 'default_language': 'nl'}).get_children_display('1000063', **kwargs) + childeren_tgn_belgie = TGNProvider({'id': 'TGN', 'default_language': 'nl'} + ).get_children_display('1000063', **kwargs) self.assertIsInstance(childeren_tgn_belgie, list) self.assertGreater(len(childeren_tgn_belgie), 0) keys_first_display = childeren_tgn_belgie[0].keys() for key in ['id', 'type', 'label', 'uri']: self.assertIn(key, keys_first_display) - self.assertIn(u'Brussels Hoofdstedelijk Gewest', [label['label'] for label in childeren_tgn_belgie]) + self.assertIn('Brussels Hoofdstedelijk Gewest', [label['label'] for label in childeren_tgn_belgie]) def test_expand(self): all_childeren_churches = AATProvider({'id': 'AAT'}).expand('300007466') @@ -192,16 +191,20 @@ def test_find_without_label(self): self.assertIsInstance(r, list) def test_find_wrong_type(self): - self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, {'type': 'collectie', 'collection': {'id': '300007466', 'depth': 'all'}}) + self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, { + 'type': 'collectie', 'collection': {'id': '300007466', 'depth': 'all'}}) def test_find_no_collection_id(self): - self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, {'type': 'collection', 'collection': {'depth': 'all'}}) + self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, { + 'type': 'collection', 'collection': {'depth': 'all'}}) def test_find_wrong_collection_depth(self): - self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, {'type': 'concept', 'collection': {'id': '300007466', 'depth': 'allemaal'}}) + self.assertRaises(ValueError, AATProvider({'id': 'AAT'}).find, { + 'type': 'concept', 'collection': {'id': '300007466', 'depth': 'allemaal'}}) def test_find_concepts_in_collection(self): - r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'concept', 'collection': {'id': '300007466', 'depth': 'all'}}) + r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'concept', + 'collection': {'id': '300007466', 'depth': 'all'}}) self.assertIsInstance(r, list) self.assertGreater(len(r), 0) for res in r: @@ -215,14 +218,16 @@ def test_find_multiple_keywords(self): assert res['type'] == 'concept' def test_find_member_concepts_in_collection(self): - r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'concept', 'collection': {'id': '300007494', 'depth': 'members'}}) + r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'concept', + 'collection': {'id': '300007494', 'depth': 'members'}}) self.assertIsInstance(r, list) self.assertGreater(len(r), 0) for res in r: assert res['type'] == 'concept' def test_find_collections_in_collection(self): - r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'collection', 'collection': {'id': '300007466', 'depth': 'all'}}) + r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'collection', + 'collection': {'id': '300007466', 'depth': 'all'}}) assert len(r) > 0 for res in r: assert res['type'] == 'collection' @@ -270,8 +275,9 @@ def test_find_concepts_kerk(self): assert res['type'] == 'concept' def test_find_member_collections_in_collection(self): - r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'collection', 'collection': {'id': '300007466', 'depth': 'members'}}) - assert len (r) > 0 + r = AATProvider({'id': 'AAT'}).find({'label': 'church', 'type': 'collection', + 'collection': {'id': '300007466', 'depth': 'members'}}) + assert len(r) > 0 for res in r: assert res['type'] == 'collection' @@ -282,14 +288,15 @@ def test_find_matches(self): assert r[0]['uri'] == 'http://vocab.getty.edu/aat/300191778' def test_find_closematches(self): - r = AATProvider({'id': 'AAT'}).find({'matches': {'uri': 'http://id.loc.gov/authorities/subjects/sh85123119', 'type': 'close'}}) + r = AATProvider({'id': 'AAT'}).find( + {'matches': {'uri': 'http://id.loc.gov/authorities/subjects/sh85123119', 'type': 'close'}}) assert len(r) == 1 assert r[0]['type'] == 'concept' assert r[0]['uri'] == 'http://vocab.getty.edu/aat/300191778' def test_find_matches_no_uri(self): with pytest.raises(ValueError): - r = AATProvider({'id': 'AAT'}).find({'matches': {'type': 'close'}}) + AATProvider({'id': 'AAT'}).find({'matches': {'type': 'close'}}) def test_answer_wrong_query(self): with pytest.raises(ProviderUnavailableException): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3bd272c..67710cd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,16 @@ -# -*- coding: utf-8 -*- import pytest import rdflib +from rdflib.namespace import SKOS +from rdflib.plugin import PluginException from skosprovider.exceptions import ProviderUnavailableException -from skosprovider_getty.utils import ( - uri_to_graph, - SubClassCollector, - GVP, - ISO -) -class TestUtils(object): +from skosprovider_getty.utils import GVP +from skosprovider_getty.utils import ISO +from skosprovider_getty.utils import SubClassCollector +from skosprovider_getty.utils import uri_to_graph + + +class TestUtils: def test_uri_to_graph(self): uri = 'http://vocab.getty.edu/aat/300007466.rdf' @@ -18,8 +19,8 @@ def test_uri_to_graph(self): def test_uri_to_graph2(self): uri = 'http://vocab.getty.edu/aat/300007466' - with pytest.raises(TypeError): - res = uri_to_graph(uri) + with pytest.raises(PluginException): + uri_to_graph(uri) def test_uri_to_graph_not_found(self): uri = 'http://vocab.getty.edu/aat55/300zzz7466.rdf' @@ -29,31 +30,27 @@ def test_uri_to_graph_not_found(self): def test_uri_to_graph_error(self): uri = 'http://teeezssst.teeteest.test/aat55/300zzz7466.rdf' with pytest.raises(ProviderUnavailableException): - res = uri_to_graph(uri) + uri_to_graph(uri) def test_get_subclasses(self): - from rdflib.namespace import SKOS subclasses = SubClassCollector(GVP) list_concept_subclasses = subclasses.get_subclasses(SKOS.Concept) assert len(list_concept_subclasses) == 8 assert SKOS.Concept in list_concept_subclasses def test_collect_subclasses_concept(self): - from rdflib.namespace import SKOS subclasses = SubClassCollector(rdflib.Namespace("http://vocab.getty.edu/ontology#")) list_concept_subclasses = subclasses.collect_subclasses(SKOS.Concept) assert len(list_concept_subclasses) == 8 assert SKOS.Concept in list_concept_subclasses def test_collect_subclasses_collection(self): - from rdflib.namespace import SKOS subclasses = SubClassCollector(rdflib.Namespace("http://vocab.getty.edu/ontology#")) list_concept_subclasses = subclasses.collect_subclasses(SKOS.Collection) assert len(list_concept_subclasses) == 5 assert SKOS.Collection in list_concept_subclasses def test_collect_subclasses_(self): - from rdflib.namespace import SKOS subclasses = SubClassCollector(GVP) list_concept_subclasses = subclasses.collect_subclasses(ISO.ThesaurusArray) assert len(list_concept_subclasses) == 4