Initial ModelsRepositoryClient

sdk/iot/azure-iot-modelsrepository/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# E501: line length (black formatting will handle)
+# W503, E203: Not PEP8 compliant (incompatible with black formatting)
+ignore = E501,W503,E203
+exclude =
+    .git,
+    __pycache__,

sdk/iot/azure-iot-modelsrepository/README.md

@@ -0,0 +1,16 @@
+# Azure IoT Models Repository Library
+
+The Azure IoT Models Repository Library for Python provides functionality for working with the Azure IoT Models Repository
+
+## Installation
+
+This package is not yet available on pip. Please install locally from source:
+
+```Shell
+python -m pip install -e <path to azure-iot-modelsrepository>
+```
+
+## Features
+
+* ### ModelsRepositoryClient
+    * Allows retrieval of model DTDLs from remote URLs or local filesystems

sdk/iot/azure-iot-modelsrepository/azure/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py

@@ -0,0 +1,18 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+# Main Client
+from .client import ModelsRepositoryClient
+
+# Constants
+from .client import (
+    DEPENDENCY_MODE_DISABLED,
+    DEPENDENCY_MODE_ENABLED,
+    DEPENDENCY_MODE_TRY_FROM_EXPANDED,
+)
+
+# Error handling
+from .resolver import ResolverError

sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py

@@ -0,0 +1,24 @@
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+
+class ChainableException(Exception):
+    """This exception stores a reference to a previous exception which has caused
+    the current one"""
+
+    def __init__(self, message=None, cause=None):
+        # By using .__cause__, this will allow typical stack trace behavior in Python 3,
+        # while still being able to operate in Python 2.
+        self.__cause__ = cause
+        super(ChainableException, self).__init__(message)
+
+    def __str__(self):
+        if self.__cause__:
+            return "{} caused by {}".format(
+                super(ChainableException, self).__repr__(), self.__cause__.__repr__()
+            )
+        else:
+            return super(ChainableException, self).__repr__()

sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py

@@ -0,0 +1,173 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import six.moves.urllib as urllib
+import re
+from azure.core import PipelineClient
+from azure.core.pipeline.transport import RequestsTransport
+from azure.core.configuration import Configuration
+from azure.core.pipeline.policies import (
+    UserAgentPolicy,
+    HeadersPolicy,
+    RetryPolicy,
+    RedirectPolicy,
+    BearerTokenCredentialPolicy,
+    ContentDecodePolicy,
+    NetworkTraceLoggingPolicy,
+    ProxyPolicy,
+)
+from . import resolver as resolver
+from . import pseudo_parser
+from .chainable_exception import ChainableException
+
+
+# Public constants exposed to consumers
+DEPENDENCY_MODE_TRY_FROM_EXPANDED = "tryFromExpanded"
+DEPENDENCY_MODE_DISABLED = "disabled"
+DEPENDENCY_MODE_ENABLED = "enabled"
+
+
+# Convention-private constants
+_DEFAULT_LOCATION = "https://devicemodels.azure.com"
+_DEFAULT_API_VERSION = "2021-02-11"
+_REMOTE_PROTOCOLS = ["http", "https"]
+
+
+class ModelsRepositoryClient(object):
+    """Client providing APIs for Models Repository operations"""
+
+    def __init__(self, repository_location=None, api_version=None, **kwargs):
+        """
+        :param str repository_location: Location of the Models Repository you wish to access.
+            This location can be a remote HTTP/HTTPS URL, or a local filesystem path.
+            If omitted, will default to using "https://devicemodels.azure.com".
+
+        :raises: ValueError if repository_location is invalid
+        """
+        repository_location = (
+            _DEFAULT_LOCATION if repository_location is None else repository_location
+        )
+        # api_version = _DEFAULT_API_VERSION if api_version is None else api_version
+
+        kwargs.setdefault("api_verison", api_version)
+
+        # NOTE: depending on how this class develops over time, may need to adjust relationship
+        # between some of these objects
+        self.fetcher = _create_fetcher(
+            location=repository_location, api_version=api_version, **kwargs
+        )
+        self.resolver = resolver.DtmiResolver(self.fetcher)
+        self.pseudo_parser = pseudo_parser.PseudoParser(self.resolver)
+
+    def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED):
+        """Retrieve a model from the Models Repository.
+
+        :param list[str]: The DTMIs for the models you wish to retrieve
+        :param str dependency_resolution : Dependency resolution mode. Possible values:
+            - "disabled": Do not resolve model dependencies
+            - "enabled": Resolve model dependencies from the repository
+            - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL
+                    document in the repository. If this is not successful, will fall back on
+                    manually resolving dependencies in the repository
+
+        :raises: ValueError if given an invalid dependency resolution mode
+        :raises: ResolverError if there is an error retreiving a model
+
+        :returns: Dictionary mapping DTMIs to models
+        :rtype: dict
+        """
+        if dependency_resolution == DEPENDENCY_MODE_DISABLED:
+            model_map = self.resolver.resolve(dtmis)
+        elif dependency_resolution == DEPENDENCY_MODE_ENABLED:
+            # Manually resolve dependencies using pseudo-parser
+            base_model_map = model_map = self.resolver.resolve(dtmis)
+            base_model_list = [model for model in base_model_map.values()]
+            model_map = self.pseudo_parser.expand(base_model_list)
+        elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED:
+            # Try to use an expanded DTDL to resolve dependencies
+            try:
+                model_map = self.resolver.resolve(dtmis, expanded_model=True)
+            except resolver.ResolverError:
+                # Fallback to manual dependency resolution
+                base_model_map = model_map = self.resolver.resolve(dtmis)
+                base_model_list = [model for model in base_model_map.items()]
+                model_map = self.pseudo_parser.expand(base_model_list)
+        else:
+            raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution))
+        return model_map
+
+
+class ModelsRepositoryClientConfiguration(Configuration):
+    """ModelsRepositoryClient-specific variant of the Azure Core Configuration for Pipelines"""
+
+    def __init__(self, **kwargs):
+        super(ModelsRepositoryClientConfiguration, self).__init__(**kwargs)
+        # NOTE: There might be some further organization to do here as it's kind of weird that
+        # the generic config (which could be used for any remote repository) always will have
+        # the default repository's api version stored. Keep this in mind when expanding the
+        # scope of the client in the future - perhaps there may need to eventually be unique
+        # configs for default repository vs. custom repository endpoints
+        self._api_version = kwargs.get("api_version", _DEFAULT_API_VERSION)
+
+
+def _create_fetcher(location, **kwargs):
+    """Return a Fetcher based upon the type of location"""
+    scheme = urllib.parse.urlparse(location).scheme
+    if scheme in _REMOTE_PROTOCOLS:
+        # HTTP/HTTPS URL
+        client = _create_pipeline_client(base_url=location, **kwargs)
+        fetcher = resolver.HttpFetcher(client)
+    elif scheme == "file":
+        # Filesystem URI
+        location = location[len("file://") :]
+        fetcher = resolver.FilesystemFetcher(location)
+    elif scheme == "" and location.startswith("/"):
+        # POSIX filesystem path
+        fetcher = resolver.FilesystemFetcher(location)
+    elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]):
+        # Web URL with protocol unspecified - default to HTTPS
+        location = "https://" + location
+        client = _create_pipeline_client(base_url=location, **kwargs)
+        fetcher = resolver.HttpFetcher(client)
+    elif scheme != "" and len(scheme) == 1 and scheme.isalpha():
+        # Filesystem path using drive letters (e.g. "C:", "D:", etc.)
+        fetcher = resolver.FilesystemFetcher(location)
+    else:
+        raise ValueError("Unable to identify location: {}".format(location))
+    return fetcher
+
+
+def _create_pipeline_client(base_url, **kwargs):
+    """Creates and returns a PipelineClient configured for the provided base_url and kwargs"""
+    transport = kwargs.get("transport", RequestsTransport(**kwargs))
+    config = _create_config(**kwargs)
+    policies = [
+        config.user_agent_policy,
+        config.headers_policy,
+        config.authentication_policy,
+        ContentDecodePolicy(),
+        config.proxy_policy,
+        config.redirect_policy,
+        config.retry_policy,
+        config.logging_policy,
+    ]
+    return PipelineClient(base_url=base_url, config=config, policies=policies, transport=transport)
+
+
+def _create_config(**kwargs):
+    """Creates and returns a ModelsRepositoryConfiguration object"""
+    config = ModelsRepositoryClientConfiguration(**kwargs)
+    config.headers_policy = kwargs.get(
+        "headers_policy", HeadersPolicy({"CustomHeader": "Value"}, **kwargs)
+    )
+    config.user_agent_policy = kwargs.get(
+        "user_agent_policy", UserAgentPolicy("ServiceUserAgentValue", **kwargs)
+    )
+    config.authentication_policy = kwargs.get("authentication_policy")
+    config.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs))
+    config.redirect_policy = kwargs.get("redirect_policy", RedirectPolicy(**kwargs))
+    config.logging_policy = kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs))
+    config.proxy_policy = kwargs.get("proxy_policy", ProxyPolicy(**kwargs))
+    return config

sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py

@@ -0,0 +1,58 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""This module contains a partial parsing implementation that is strictly
+scoped for parsing dependencies. Ideally, this would be made obsolete by
+a full Python parser implementation, as it is supposed to be a single source
+of truth for the DTDL model specifications.
+
+Note that this implementation is not representative of what an eventual full
+parser implementation would necessarily look like from an API perspective
+"""
+
+
+class PseudoParser(object):
+    def __init__(self, resolver):
+        self.resolver = resolver
+
+    def expand(self, models):
+        """"""
+        expanded_map = {}
+        for model in models:
+            expanded_map[model["@id"]] = model
+            self._expand(model, expanded_map)
+        return expanded_map
+
+    def _expand(self, model, model_map):
+        dependencies = get_dependency_list(model)
+        dependencies_to_resolve = [
+            dependency for dependency in dependencies if dependency not in model_map
+        ]
+
+        if dependencies_to_resolve:
+            resolved_dependency_map = self.resolver.resolve(dependencies_to_resolve)
+            model_map.update(resolved_dependency_map)
+            for dependency_model in resolved_dependency_map.items():
+                self._expand(dependency_model, model_map)
+
+
+def get_dependency_list(model):
+    """Return a list of DTMIs for model dependencies"""
+    if "contents" in model:
+        components = [item["schema"] for item in model["contents"] if item["@type"] == "Component"]
+    else:
+        components = []
+
+    if "extends" in model:
+        # Models defined in a DTDL can implement extensions of up to two interfaces
+        if isinstance(model["extends"], list):
+            interfaces = model["extends"]
+        else:
+            interfaces = [model["extends"]]
+    else:
+        interfaces = []
+
+    dependencies = components + interfaces
+    return dependencies