Merge pull request #1244 from google/google_sync

Google sync

CHANGELOG

@@ -1,3 +1,14 @@
+Version 2022.06.30:
+
+Updates:
+* Add the option to show kythe facts in the debug indexer.
+* Add a command-line argument `pickle-metadata` to pass metadata to pytype to
+  store during serialization to a pickle file.
+
+Bug fixes:
+* Make slight improvements to --overriding-default-value-checks.
+* Preserve comments when preprocessing source code.
+
 Version 2022.06.23:
 
 Updates:

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2022.06.23'
+__version__ = '2022.06.30'

pytype/config.py

@@ -241,6 +241,11 @@ def add_pickle_options(o):
       "--precompiled-builtins", action="store",
       dest="precompiled_builtins", default=None,
       help="Use the supplied file as precompiled builtins pyi.")
+  o.add_argument(
+      "--pickle-metadata", type=str, action="store",
+      dest="pickle_metadata", default=None,
+      help=("Comma separated list of metadata strings to be saved in the "
+            "pickled file."))
 
 
 def add_infrastructure_options(o):
@@ -645,6 +650,12 @@ def _parse_arguments(self, arguments):
       self.error("Argument %r is not a pair of non-empty file names "
                  "separated by %r" % (item, os.pathsep))
 
+  def _store_pickle_metadata(self, pickle_metadata):
+    if pickle_metadata:
+      self.output_options.pickle_metadata = pickle_metadata.split(",")
+    else:
+      self.output_options.pickle_metadata = []
+
 
 def _set_verbosity(verbosity, timestamp_logs, debug_logs):
   """Set the logging verbosity."""

pytype/config_test.py

@@ -230,6 +230,11 @@ def test_custom_open_function(self):
     options = config.Options.create(open_function=open_function)
     self.assertIs(options.open_function, open_function)
 
+  def test_pickle_metadata(self):
+    input_options = types.SimpleNamespace(pickle_metadata="meta,data",)
+    config.Postprocessor({"pickle_metadata"}, input_options).process()
+    self.assertSequenceEqual(input_options.pickle_metadata, ["meta", "data"])
+
 
 if __name__ == "__main__":
   unittest.main()

pytype/io.py

@@ -251,7 +251,8 @@ def write_pickle(ast, options, loader=None):
     if not pytd_utils.ASTeq(ast1, ast2):
       raise AssertionError()
   serialize_ast.StoreAst(ast, options.output, options.open_function,
-                         is_package=options.module_name.endswith(".__init__"))
+                         is_package=options.module_name.endswith(".__init__"),
+                         metadata=options.pickle_metadata)
 
 
 def print_error_doc_url(errorlog):

pytype/pytd/serialize_ast.py

@@ -63,7 +63,9 @@ class SerializableTupleClass(NamedTuple):
   dependencies: List[Tuple[str, Set[str]]]
   late_dependencies: List[Tuple[str, Set[str]]]
   class_type_nodes: Optional[List[pytd.ClassType]]
-  is_package: bool
+  is_package: bool  # TODO(rechen): remove this, can be computed from src_path
+  src_path: Optional[str]
+  metadata: List[str]
 
 
 class SerializableAst(SerializableTupleClass):
@@ -83,11 +85,15 @@ class SerializableAst(SerializableTupleClass):
       visited and have their .cls set. If this attribute is None the whole AST
       will be visited and all found ClassType instances will have their .cls
       set.
+    is_package: True if the original source file was a package __init__.
+    src_path: Optionally, the filepath of the original source file.
+    metadata: A list of arbitrary string-encoded metadata.
   """
   Replace = SerializableTupleClass._replace  # pylint: disable=no-member,invalid-name
 
 
-def StoreAst(ast, filename=None, open_function=open, is_package=False):
+def StoreAst(ast, filename=None, open_function=open, is_package=False,
+             src_path=None, metadata=None):
   """Loads and stores an ast to disk.
 
   Args:
@@ -96,6 +102,8 @@ def StoreAst(ast, filename=None, open_function=open, is_package=False):
       function instead returns the pickled string.
     open_function: A custom file opening function.
     is_package: Whether the module with the given ast is a package.
+    src_path: Optionally, the filepath of the original source file.
+    metadata: A list of arbitrary string-encoded metadata.
 
   Returns:
     The pickled string, if no filename was given. (None otherwise.)
@@ -116,10 +124,15 @@ def StoreAst(ast, filename=None, open_function=open, is_package=False):
   indexer = FindClassTypesVisitor()
   ast.Visit(indexer)
   ast = ast.Visit(visitors.CanonicalOrderingVisitor())
+
+  metadata = metadata or []
+
   return pytd_utils.SavePickle(
       SerializableAst(
           ast, sorted(dependencies.items()), sorted(late_dependencies.items()),
-          sorted(indexer.class_type_nodes), is_package=is_package),
+          sorted(indexer.class_type_nodes), is_package=is_package,
+          src_path=src_path, metadata=metadata,
+      ),
       filename, open_function=open_function)
 
 

pytype/pytd/serialize_ast_test.py

@@ -16,10 +16,14 @@ class SerializeAstTest(test_base.UnitTest):
 
   def _store_ast(
       self, temp_dir, module_name, pickled_ast_filename, ast=None, loader=None,
-      is_package=False):
+      is_package=False, metadata=None):
     if not (ast and loader):
       ast, loader = self._get_ast(temp_dir=temp_dir, module_name=module_name)
-    serialize_ast.StoreAst(ast, pickled_ast_filename, is_package=is_package)
+    serialize_ast.StoreAst(
+        ast,
+        pickled_ast_filename,
+        is_package=is_package,
+        metadata=metadata)
     module_map = {name: module.ast
                   for name, module in loader._modules.items()}
 
@@ -287,6 +291,14 @@ def test_function(self):
       signature, = f.type.signatures
       self.assertIsNotNone(signature.return_type.cls)
 
+  def test_pickle_metadata(self):
+    with file_utils.Tempdir() as d:
+      module_name = "module1"
+      pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
+      self._store_ast(
+          d, module_name, pickled_ast_filename, metadata=["meta", "data"])
+      serialized_ast = pytd_utils.LoadPickle(pickled_ast_filename)
+      self.assertSequenceEqual(serialized_ast.metadata, ["meta", "data"])
 
 if __name__ == "__main__":
   unittest.main()

pytype/tools/xref/parse_args.py

@@ -34,6 +34,9 @@ def add_kythe_field(parser, field):
   parser.add_argument("--show-types", action="store_true",
                       dest="show_types", default=None,
                       help="Display inferred types.")
+  parser.add_argument("--show-kythe", action="store_true",
+                      dest="show_kythe", default=None,
+                      help="Display kythe facts.")
   # TODO(b/124802213): There should be a cleaner way to do this.
   parser.add_argument(
       "--imports_info", type=str, action="store",