From 299d8ecf20f6b7d644d82060a4dd76c60ebf00c3 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Thu, 5 Dec 2024 11:13:45 -0800 Subject: [PATCH 1/2] jom (parallel nmake) build for openssl windows Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1084 openssl build on windows is slow due to nmake being single threaded fortunately the Qt developers had the same problem and produced jom - a nmake compatible make that adds the /j flag add a jom manifest and use it for the openssl build on windows Reviewed By: bigfootjon Differential Revision: D66818562 --- build/fbcode_builder/getdeps/builder.py | 14 +++++++++++--- build/fbcode_builder/manifests/jom | 15 +++++++++++++++ build/fbcode_builder/manifests/openssl | 1 + 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 build/fbcode_builder/manifests/jom diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 10bd2a62a62..6bd5f526e7f 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -1165,9 +1165,14 @@ def _build(self, reconfigure) -> None: perl = typing.cast(str, path_search(env, "perl", "perl")) make_j_args = [] + extra_args = [] if self.build_opts.is_windows(): - make = "nmake.exe" + # jom is compatible with nmake, adds the /j argument for parallel build + make = "jom.exe" + make_j_args = ["/j%s" % self.num_jobs] args = ["VC-WIN64A-masm", "-utf-8"] + # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS" + extra_args = ["/FS"] elif self.build_opts.is_darwin(): make = "make" make_j_args = ["-j%s" % self.num_jobs] @@ -1200,11 +1205,14 @@ def _build(self, reconfigure) -> None: "no-unit-test", "no-tests", ] + + extra_args ) + # show the config produced + self._run_cmd([perl, "configdata.pm", "--dump"], env=env) make_build = [make] + make_j_args - self._run_cmd(make_build) + self._run_cmd(make_build, env=env) make_install = [make, "install_sw", "install_ssldirs"] - self._run_cmd(make_install) + self._run_cmd(make_install, env=env) class Boost(BuilderBase): diff --git a/build/fbcode_builder/manifests/jom b/build/fbcode_builder/manifests/jom new file mode 100644 index 00000000000..effecab67a1 --- /dev/null +++ b/build/fbcode_builder/manifests/jom @@ -0,0 +1,15 @@ +# jom is compatible with MSVC nmake, but adds the /j argment which +# speeds up openssl build a lot +[manifest] +name = jom + +# see https://download.qt.io/official_releases/jom/changelog.txt for latest version +[download.os=windows] +url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip +sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b + +[build.os=windows] +builder = nop + +[install.files.os=windows] +. = bin diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index 7dd40727cc2..ebd680e7e1e 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -31,4 +31,5 @@ builder = openssl subdir = openssl-3.0.15 [dependencies.os=windows] +jom perl From 3e3b9dce3d073f9489d8e9f7772f8bf6d91bf999 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Thu, 5 Dec 2024 11:13:45 -0800 Subject: [PATCH 2/2] speed up file copy on windows Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1085 getdeps windows file copy is very slow, speed it up Reviewed By: bigfootjon Differential Revision: D66830544 --- build/fbcode_builder/getdeps/builder.py | 5 ++- build/fbcode_builder/getdeps/cargo.py | 3 +- build/fbcode_builder/getdeps/copytree.py | 47 ++++++++++++++++-------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 6bd5f526e7f..83916fde947 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -18,6 +18,7 @@ from shlex import quote as shellquote from typing import Optional +from .copytree import simple_copytree from .dyndeps import create_dyn_dep_munger from .envfuncs import add_path_entry, Env, path_search from .fetcher import copy_if_different @@ -1329,7 +1330,7 @@ def build(self, reconfigure: bool) -> None: os.makedirs(dest_parent) if os.path.isdir(full_src): if not os.path.exists(full_dest): - shutil.copytree(full_src, full_dest) + simple_copytree(full_src, full_dest) else: shutil.copyfile(full_src, full_dest) shutil.copymode(full_src, full_dest) @@ -1341,7 +1342,7 @@ def build(self, reconfigure: bool) -> None: os.chmod(full_dest, st.st_mode | stat.S_IXUSR) else: if not os.path.exists(self.inst_dir): - shutil.copytree(self.src_dir, self.inst_dir) + simple_copytree(self.src_dir, self.inst_dir) class SqliteBuilder(BuilderBase): diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 0e0e0ddfe0b..5bb2ada85c2 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -13,6 +13,7 @@ import typing from .builder import BuilderBase +from .copytree import simple_copytree if typing.TYPE_CHECKING: from .buildopts import BuildOptions @@ -79,7 +80,7 @@ def recreate_dir(self, src, dst) -> None: os.remove(dst) else: shutil.rmtree(dst) - shutil.copytree(src, dst) + simple_copytree(src, dst) def cargo_config_file(self): build_source_dir = self.build_dir diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py index 2297bd3aa80..6815f74c898 100644 --- a/build/fbcode_builder/getdeps/copytree.py +++ b/build/fbcode_builder/getdeps/copytree.py @@ -10,6 +10,7 @@ import subprocess from .platform import is_windows +from .runcmd import run_cmd PREFETCHED_DIRS = set() @@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None: PREFETCHED_DIRS.add(dirpath) -# pyre-fixme[9]: ignore has type `bool`; used as `None`. -def copytree(src_dir, dest_dir, ignore: bool = None): - """Recursively copy the src_dir to the dest_dir, filtering - out entries using the ignore lambda. The behavior of the - ignore lambda must match that described by `shutil.copytree`. - This `copytree` function knows how to prefetch data when - running in an eden repo. - TODO: I'd like to either extend this or add a variant that - uses watchman to mirror src_dir into dest_dir. - """ - prefetch_dir_if_eden(src_dir) - # pyre-fixme[6]: For 3rd param expected - # `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]], - # typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`. - return shutil.copytree(src_dir, dest_dir, ignore=ignore) +def simple_copytree(src_dir, dest_dir, symlinks=False): + """A simple version of shutil.copytree() that can delegate to native tools if faster""" + if is_windows(): + os.makedirs(dest_dir, exist_ok=True) + cmd = [ + "robocopy.exe", + src_dir, + dest_dir, + # copy directories, including empty ones + "/E", + # Ignore Extra files in destination + "/XX", + # enable parallel copy + "/MT", + # be quiet + "/NFL", + "/NDL", + "/NJH", + "/NJS", + "/NP", + ] + if symlinks: + cmd.append("/SL") + # robocopy exits with code 1 if it copied ok, hence allow_fail + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility + exit_code = run_cmd(cmd, allow_fail=True) + if exit_code > 1: + raise subprocess.CalledProcessError(exit_code, cmd) + return dest_dir + else: + return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)