[Python APIView] CrossLangugageDefintionId implementation (#2781)

* Refactor APIView primitives to use snake_case instead of PascalCase. Add
apiview_mapping file.

* Implement CrossLanguageDefinitionId for classes and functions.

* Code review feedback.

* Support --mapping-path arg for parsing WHL files.

packages/python-packages/api-stub-generator/CHANGELOG.md

@@ -6,6 +6,8 @@ Added support to parse defaults from docstrings. Example
   syntax: "A value, defaults to foo."
   Also supports older, non-recommended syntax, such as:
   "A value. Default value is foo."
+Add support for "CrossLanguageDefinitionId" and a --mapping-path
+  parameter to supply the necessary mapping file.
 
 ## Version 0.2.9 (Unreleased)
 Fixed issue where Python 3-style type hints stopped displaying

packages/python-packages/api-stub-generator/apistub/__init__.py

@@ -29,7 +29,7 @@ def console_entry_point():
     # Generate JSON file name if outpath doesn't have json file name
     if not out_file_path.endswith(".json"):
         out_file_path = os.path.join(
-            stub_generator.out_path, "{0}_python.json".format(apiview.Name)
+            stub_generator.out_path, "{0}_python.json".format(apiview.name)
         )
     with open(out_file_path, "w") as json_file:
         json_file.write(json_tokens)

packages/python-packages/api-stub-generator/apistub/_apiview.py

@@ -1,4 +1,3 @@
-import json
 from json import JSONEncoder
 import logging
 import re
@@ -10,6 +9,7 @@
 from ._token_kind import TokenKind
 from ._version import VERSION
 from ._diagnostic import Diagnostic
+from ._metadata_map import MetadataMap
 
 JSON_FIELDS = ["Name", "Version", "VersionString", "Navigation", "Tokens", "Diagnostics", "PackageName", "Language"]
 
@@ -25,29 +25,29 @@ class ApiView:
     """Entity class that holds API view for all namespaces within a package
     :param NodeIndex: nodeindex
     :param str: pkg_name
-    :param str: pkg_version
     :param str: ver_string
     """
 
-    def __init__(self, nodeindex, pkg_name="", pkg_version="", namespace = ""):
-        self.Name = pkg_name
-        self.Version = 0
-        self.VersionString = ""
-        self.Language = "Python"
-        self.Tokens = []
-        self.Navigation = []
-        self.Diagnostics = []
+    def __init__(self, nodeindex, pkg_name="", namespace = "", metadata_map=None):
+        self.name = pkg_name
+        self.version = 0
+        self.version_string = ""
+        self.language = "Python"
+        self.tokens = []
+        self.navigation = []
+        self.diagnostics = []
         self.indent = 0    
         self.namespace = namespace
         self.nodeindex = nodeindex
-        self.PackageName = pkg_name
+        self.package_name = pkg_name
+        self.metadata_map = metadata_map or MetadataMap("")
         self.add_token(Token("", TokenKind.SkipDiffRangeStart))
         self.add_literal(HEADER_TEXT)
         self.add_token(Token("", TokenKind.SkipDiffRangeEnd))
         self.set_blank_lines(2)
 
     def add_token(self, token):
-        self.Tokens.append(token)
+        self.tokens.append(token)
 
     def begin_group(self, group_name=""):
         """Begin a new group in API view by shifting to right
@@ -73,7 +73,7 @@ def add_newline(self):
             Cannot be used to inject blank lines.
         """
         # don't add newline if it already is in place
-        if self.Tokens[-1].Kind != TokenKind.Newline:
+        if self.tokens[-1].kind != TokenKind.Newline:
             self.add_token(Token("", TokenKind.Newline))
 
     def set_blank_lines(self, count):
@@ -83,8 +83,8 @@ def set_blank_lines(self, count):
         """
         # count the number of trailing newlines
         newline_count = 0
-        for token in self.Tokens[::-1]:
-            if token.Kind == TokenKind.Newline:
+        for token in self.tokens[::-1]:
+            if token.kind == TokenKind.Newline:
                 newline_count += 1
             else:
                 break
@@ -97,7 +97,7 @@ def set_blank_lines(self, count):
             # if there are too many newlines, remove some
             excess = newline_count - (count + 1)
             for _ in range(excess):
-                self.Tokens.pop()
+                self.tokens.pop()
 
     def add_punctuation(self, value, prefix_space=False, postfix_space=False):
         if prefix_space:
@@ -108,12 +108,14 @@ def add_punctuation(self, value, prefix_space=False, postfix_space=False):
 
     def add_line_marker(self, text):
         token = Token("", TokenKind.LineIdMarker)
-        token.set_definition_id(text)
+        token.definition_id = text
         self.add_token(token)
 
-    def add_text(self, id, text):
+    def add_text(self, id, text, add_cross_language_id=False):
         token = Token(text, TokenKind.Text)
-        token.DefinitionId = id
+        token.definition_id = id
+        if add_cross_language_id:
+            token.cross_language_definition_id = self.metadata_map.cross_language_map.get(id, None)
         self.add_token(token)
 
     def add_keyword(self, keyword, prefix_space=False, postfix_space=False):
@@ -145,17 +147,17 @@ def _add_token_for_type_name(self, type_name, line_id = None):
         logging.debug("Generating tokens for type name {}".format(type_name))
         token = Token(type_name, TokenKind.TypeName)
         type_full_name = type_name[1:] if type_name.startswith("~") else type_name
-        token.set_value(type_full_name.split(".")[-1])
+        token.value = type_full_name.split(".")[-1]
         navigate_to_id = self.nodeindex.get_id(type_full_name)
         if navigate_to_id:
-            token.NavigateToId = navigate_to_id
+            token.navigate_to_id = navigate_to_id
         elif type_name.startswith("~") and line_id:
             # Check if type name is importable. If type name is incorrect in docstring then it wont be importable
             # If type name is importable then it's a valid type name. Source link wont be available if type is from 
             # different package
             if not is_valid_type_name(type_full_name):
                 # Navigation ID is missing for internal type, add diagnostic error
-                self.add_diagnostic(SOURCE_LINK_NOT_AVAILABLE.format(token.Value), line_id)            
+                self.add_diagnostic(SOURCE_LINK_NOT_AVAILABLE.format(token.value), line_id)            
         self.add_token(token)
 
 
@@ -183,12 +185,12 @@ def _add_type_token(self, type_name, line_id = None):
 
 
     def add_diagnostic(self, text, line_id):
-        self.Diagnostics.append(Diagnostic(line_id, text))
+        self.diagnostics.append(Diagnostic(line_id, text))
 
 
     def add_member(self, name, id):
         token = Token(name, TokenKind.MemberName)
-        token.DefinitionId = id
+        token.definition_id = id
         self.add_token(token)
 
 
@@ -201,40 +203,43 @@ def add_literal(self, value):
 
 
     def add_navigation(self, navigation):
-        self.Navigation.append(navigation)
-
+        self.navigation.append(navigation)
 
 class APIViewEncoder(JSONEncoder):
     """Encoder to generate json for APIview object
     """
 
+    def _snake_to_pascal(self, text: str) -> str:
+        return text.replace("_", " ").title().replace(" ", "")
+
+    def _pascal_to_snake(self, text: str) -> str:
+        results = "_".join([x.lower() for x in re.findall('[A-Z][^A-Z]*', text)])
+        return results
+
     def default(self, obj):
         obj_dict = {}
-        if (
-            isinstance(obj, ApiView)
-            or isinstance(obj, Token)
-            or isinstance(obj, Navigation)
-            or isinstance(obj, NavigationTag)
-            or isinstance(obj, Diagnostic)
-        ):            
+        if isinstance(obj, (ApiView, Token, Navigation, NavigationTag, Diagnostic)):            
             # Remove fields in APIview that are not required in json
             if isinstance(obj, ApiView):
                 for key in JSON_FIELDS:
-                    if key in obj.__dict__:
-                        obj_dict[key] = obj.__dict__[key]
+                    snake_key = self._pascal_to_snake(key)
+                    if snake_key in obj.__dict__:
+                        obj_dict[key] = obj.__dict__[snake_key]
             elif isinstance(obj, Token):
-                obj_dict = obj.__dict__
+                obj_dict = {self._snake_to_pascal(k):v for k, v in obj.__dict__.items()}
                 # Remove properties from serialization to reduce size if property is not set
-                if not obj.DefinitionId:
+                if not obj.definition_id:
                     del obj_dict["DefinitionId"]
-                if not obj.NavigateToId:
+                if not obj.navigate_to_id:
                     del obj_dict["NavigateToId"]
+                if not obj.cross_language_definition_id:
+                    del obj_dict["CrossLanguageDefinitionId"]
             elif isinstance(obj, Diagnostic):
-                obj_dict = obj.__dict__
-                if not obj.HelpLinkUri:
+                obj_dict = {self._snake_to_pascal(k):v for k, v in obj.__dict__.items()}
+                if not obj.help_link_uri:
                     del obj_dict["HelpLinkUri"]
             else:
-                obj_dict = obj.__dict__
+                obj_dict = {self._snake_to_pascal(k):v for k, v in obj.__dict__.items()}
 
             return obj_dict
         elif isinstance(obj, TokenKind) or isinstance(obj, Kind):
@@ -249,7 +254,7 @@ def default(self, obj):
 
 class NavigationTag:
     def __init__(self, kind):
-        self.TypeKind = kind
+        self.type_kind = kind
 
 
 class Kind:
@@ -264,16 +269,13 @@ class Navigation:
     """Navigation model to be added into tokens files. List of Navigation object represents the tree panel in tool"""
 
     def __init__(self, text, nav_id):
-        self.Text = text
-        self.NavigationId = nav_id
-        self.ChildItems = []
-        self.Tags = None
-
-    def set_tag(self, tag):
-        self.Tags = tag
+        self.text = text
+        self.navigation_id = nav_id
+        self.child_items = []
+        self.tags = None
 
     def add_child(self, child):
-        self.ChildItems.append(child)
+        self.child_items.append(child)
 
 
 def is_valid_type_name(type_name):

packages/python-packages/api-stub-generator/apistub/_diagnostic.py

@@ -1,17 +1,9 @@
-import logging
-
 class Diagnostic:
     id_counter = 1
 
     def __init__(self, target_id, message):
-        self.DiagnosticId = "AZ_PY_{}".format(Diagnostic.id_counter)
+        self.diagnostic_id = "AZ_PY_{}".format(Diagnostic.id_counter)
         Diagnostic.id_counter+=1
-        self.Text = message
-        self.HelpLinkUri = ""
-        self.TargetId = target_id
-
-    def set_text(self, text):
-        self.Text = text
-
-    def set_helplink(self, helplink):
-        self.HelpLinkUri = helplink
+        self.text = message
+        self.help_link_uri = ""
+        self.target_id = target_id

packages/python-packages/api-stub-generator/apistub/_metadata_map.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import json
+import os
+
+MAPPING_FILE_NAME = "apiview_mapping.json"
+
+"""
+Loads metadata from the mapping file for use
+by the stub generator.
+"""
+class MetadataMap:
+
+    def __init__(self, pkg_path, mapping_path=None):
+        if not mapping_path:
+            if pkg_path.endswith(".whl") or pkg_path.endswith(".zip"):
+                self.cross_language_map = {}
+                return
+            mapping_path = os.path.join(pkg_path, MAPPING_FILE_NAME)
+
+        try:
+            with open(mapping_path, "r") as json_file:
+                mapping = json.load(json_file)
+                self.cross_language_map = mapping.get("CrossLanguageDefinitionId", {})
+        except OSError:
+            self.cross_language_map = {}
+            return

packages/python-packages/api-stub-generator/apistub/_stub_generator.py

@@ -10,23 +10,19 @@
 import os
 import argparse
 
-import inspect
 import io
 import importlib
-import json
 import logging
-import pkgutil
 import shutil
 import ast
 import textwrap
-import re
-import typing
 import tempfile
 from subprocess import check_call
 import zipfile
 
 
 from apistub._apiview import ApiView, APIViewEncoder, Navigation, Kind, NavigationTag
+from apistub._metadata_map import MetadataMap
 
 INIT_PY_FILE = "__init__.py"
 TOP_LEVEL_WHEEL_FILE = "top_level.txt"
@@ -52,6 +48,11 @@ def __init__(self):
             default=os.getcwd(),
             help=("Path to generate json file with parsed tokens"),
         )
+        parser.add_argument(
+            "--mapping-path",
+            default=None,
+            help=("Path to the 'apiview_mapping.json' file.")
+        )
         parser.add_argument(
             "--verbose",
             help=("Enable verbose logging"),
@@ -70,7 +71,6 @@ def __init__(self):
             "--filter-namespace",
             help=("Generate Api view only for a specific namespace"),
         )
-
 
         args = parser.parse_args()
         if not os.path.exists(args.pkg_path):
@@ -80,18 +80,17 @@ def __init__(self):
             logging.error("Temp path [{0}] is invalid".format(args.temp_path))
             exit(1)
 
-
         self.pkg_path = args.pkg_path
         self.temp_path = args.temp_path
         self.out_path = args.out_path
+        self.mapping_path = args.mapping_path
         self.hide_report = args.hide_report
         if args.verbose:
             logging.getLogger().setLevel(logging.DEBUG)
 
         self.filter_namespace = ''
         if args.filter_namespace:
             self.filter_namespace = args.filter_namespace
-
 
     def generate_tokens(self):
         # Extract package to temp directory if it is wheel or sdist
@@ -116,7 +115,7 @@ def generate_tokens(self):
 
         logging.debug("Generating tokens")
         apiview = self._generate_tokens(pkg_root_path, pkg_name, version, namespace)
-        if apiview.Diagnostics:
+        if apiview.diagnostics:
             # Show error report in console
             if not self.hide_report:
                 print("************************** Error Report **************************")
@@ -176,7 +175,8 @@ def _generate_tokens(self, pkg_root_path, package_name, version, namespace):
 
         self.module_dict = {}
         nodeindex = NodeIndex()
-        apiview = ApiView(nodeindex, package_name, version, namespace)
+        mapping = MetadataMap(pkg_root_path, mapping_path=self.mapping_path)
+        apiview = ApiView(nodeindex, package_name, namespace, metadata_map=mapping)
         modules = self._find_modules(pkg_root_path)
         logging.debug("Modules to generate tokens: {}".format(modules))
 
@@ -192,7 +192,7 @@ def _generate_tokens(self, pkg_root_path, package_name, version, namespace):
 
         # Create navigation info to navigate within APIreview tool
         navigation = Navigation(package_name, None)
-        navigation.set_tag(NavigationTag(Kind.type_package))
+        navigation.tag = NavigationTag(Kind.type_package)
         apiview.add_navigation(navigation)
 
         # Generate tokens