From b8208726d7561cfce323f333789bb9de7dd8acc1 Mon Sep 17 00:00:00 2001 From: Ulf Magnusson Date: Tue, 29 Oct 2019 08:48:05 +0100 Subject: [PATCH] scripts: edtlib/extract_dts_includes.py: Speed up 2x+ with yaml.CLoader Use the LibYAML-based yaml.CLoader if available instead of yaml.Loader, which is written in Python and slow. See https://pyyaml.org/wiki/PyYAMLDocumentation. This speeds up gen_defines.py from 0.2s to 0.07s on my system, for -DBOARD=hifive1. It should also make scripts/kconfig/kconfig.py faster, because it indirectly uses edtlib via scripts/kconfig/kconfigfunctions.py. yaml.CLoader seems to be available out of the box when installing with pip on Ubuntu at least. Helps with https://github.com/zephyrproject-rtos/zephyr/issues/20104. Signed-off-by: Ulf Magnusson --- scripts/dts/edtlib.py | 22 +++++++++++++--------- scripts/dts/extract_dts_includes.py | 17 ++++++++++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/scripts/dts/edtlib.py b/scripts/dts/edtlib.py index d686e8da9605..89e132cd09d0 100644 --- a/scripts/dts/edtlib.py +++ b/scripts/dts/edtlib.py @@ -27,6 +27,12 @@ import sys import yaml +try: + # Use the C LibYAML parser if available, rather than the Python parser. + # This makes e.g. gen_defines.py more than twice as fast. + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader from dtlib import DT, DTError, to_num, to_nums, TYPE_EMPTY, TYPE_NUMS, \ TYPE_PHANDLE, TYPE_PHANDLES_AND_NUMS @@ -168,10 +174,10 @@ def _init_compat2binding(self, bindings_dirs): # Only bindings for 'compatible' strings that appear in the devicetree # are loaded. - # Add legacy '!include foo.yaml' handling. Do - # yaml.Loader.add_constructor() instead of yaml.add_constructor() to be - # compatible with both version 3.13 and version 5.1 of PyYAML. - yaml.Loader.add_constructor("!include", _binding_include) + # Add legacy '!include foo.yaml' handling. Do Loader.add_constructor() + # instead of yaml.add_constructor() to be compatible with both version + # 3.13 and version 5.1 of PyYAML. + Loader.add_constructor("!include", _binding_include) dt_compats = _dt_compats(self._dt) # Searches for any 'compatible' string mentioned in the devicetree @@ -188,9 +194,7 @@ def _init_compat2binding(self, bindings_dirs): contents = f.read() # As an optimization, skip parsing files that don't contain any of - # the .dts 'compatible' strings, which should be reasonably safe. - # This optimization shaves 5+ seconds off 'cmake' configuration - # time on my system. Using yaml.CParser would probably help too. + # the .dts 'compatible' strings, which should be reasonably safe if not dt_compats_search(contents): continue @@ -201,7 +205,7 @@ def _init_compat2binding(self, bindings_dirs): try: # Parsed PyYAML output (Python lists/dictionaries/strings/etc., # representing the file) - binding = yaml.load(contents, Loader=yaml.Loader) + binding = yaml.load(contents, Loader=Loader) except yaml.YAMLError as e: self._warn("'{}' appears in binding directories but isn't " "valid YAML: {}".format(binding_path, e)) @@ -367,7 +371,7 @@ def _load_binding(self, fname): with open(paths[0], encoding="utf-8") as f: return self._merge_included_bindings( - yaml.load(f, Loader=yaml.Loader), + yaml.load(f, Loader=Loader), paths[0]) def _init_nodes(self): diff --git a/scripts/dts/extract_dts_includes.py b/scripts/dts/extract_dts_includes.py index 91d7dcaa0ccd..d95e78d1786d 100755 --- a/scripts/dts/extract_dts_includes.py +++ b/scripts/dts/extract_dts_includes.py @@ -16,10 +16,17 @@ import os, fnmatch import re -import yaml import argparse from collections import defaultdict +import yaml +try: + # Use the C LibYAML parser if available, rather than the Python parser. + # It's much faster. + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + from devicetree import parse_file from extract.globals import * import extract.globals @@ -336,7 +343,7 @@ def load_bindings(root, binding_dirs): compats = [] # Add '!include foo.yaml' handling - yaml.Loader.add_constructor('!include', yaml_include) + Loader.add_constructor('!include', yaml_include) # Code below is adapated from edtlib.py @@ -353,7 +360,7 @@ def load_bindings(root, binding_dirs): if not dt_compats_search(contents): continue - binding = yaml.load(contents, Loader=yaml.Loader) + binding = yaml.load(contents, Loader=Loader) binding_compats = _binding_compats(binding) if not binding_compats: @@ -361,7 +368,7 @@ def load_bindings(root, binding_dirs): with open(file, 'r', encoding='utf-8') as yf: binding = merge_included_bindings(file, - yaml.load(yf, Loader=yaml.Loader)) + yaml.load(yf, Loader=Loader)) for compat in binding_compats: if compat not in compats: @@ -456,7 +463,7 @@ def load_binding_file(fname): "!include statement: {}".format(fname, filepaths)) with open(filepaths[0], 'r', encoding='utf-8') as f: - return yaml.load(f, Loader=yaml.Loader) + return yaml.load(f, Loader=Loader) def yaml_inc_error(msg):