From cac267c102dd79cc73285d3f63556b0f18051a83 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 24 Mar 2023 12:04:03 -0400 Subject: [PATCH] Add pkgimage.mk to auto-parallelize pkgimage caching for stdlibs (#48069) --- Makefile | 12 ++-- contrib/cache_stdlibs.jl | 53 ---------------- pkgimage.mk | 122 ++++++++++++++++++++++++++++++++++++ stdlib/.gitignore | 1 + stdlib/LLD_jll/Project.toml | 2 - 5 files changed, 128 insertions(+), 62 deletions(-) delete mode 100644 contrib/cache_stdlibs.jl create mode 100644 pkgimage.mk diff --git a/Makefile b/Makefile index 84f08f47761ca..c6c1717732056 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ all: debug release DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/gcext test/llvmpasses) -BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk +BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk $(BUILDROOT)/pkgimage.mk DIRS += $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @# add Makefiles to the build directories for convenience (pointing back to the source location of each) @@ -104,7 +104,10 @@ julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-sysimg-ji julia julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest julia-libllvmcalltest julia-base-cache -debug release : % : julia-% +stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$* + +debug release : % : julia-% stdlibs-cache-% docs: julia-sysimg-$(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no' @@ -349,11 +352,6 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif - # Cache stdlibs - @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no $(JULIAHOME)/contrib/cache_stdlibs.jl) - # CI uses `--check-bounds=yes` which impacts the cache flags - @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes $(JULIAHOME)/contrib/cache_stdlibs.jl) - # Copy in all .jl sources as well mkdir -p $(DESTDIR)$(datarootdir)/julia/base $(DESTDIR)$(datarootdir)/julia/test cp -R -L $(JULIAHOME)/base/* $(DESTDIR)$(datarootdir)/julia/base diff --git a/contrib/cache_stdlibs.jl b/contrib/cache_stdlibs.jl deleted file mode 100644 index 37f002e043dde..0000000000000 --- a/contrib/cache_stdlibs.jl +++ /dev/null @@ -1,53 +0,0 @@ -# Stdlibs sorted in dependency, then alphabetical, order by contrib/print_sorted_stdlibs.jl -# Run with the `--exclude-sysimage` option to filter out all packages included in the system image -stdlibs = [ - # No dependencies - - # 1-depth packages - :GMP_jll, - :LLVMLibUnwind_jll, - :LibUV_jll, - :LibUnwind_jll, - :MbedTLS_jll, - :OpenLibm_jll, - :PCRE2_jll, - :Zlib_jll, - :dSFMT_jll, - :libLLVM_jll, - :DelimitedFiles, - - # 2-depth packages - :LibSSH2_jll, - :MPFR_jll, - - # 3-depth packages - :LibGit2_jll, - - # 4-depth packages - :SparseArrays, - - # 7-depth packages - :LLD_jll, - :SuiteSparse, - - # 9-depth packages - :Statistics, - :SuiteSparse_jll, -] - -depot = abspath(Sys.BINDIR, "..", "share", "julia") - -if haskey(ENV, "JULIA_CPU_TARGET") - target = ENV["JULIA_CPU_TARGET"] -else - target = "native" -end - -@info "Caching stdlibrary to" depot target -empty!(Base.DEPOT_PATH) -push!(Base.DEPOT_PATH, depot) - -for pkg in stdlibs - pkgid = Base.identify_package(string(pkg)) - Base.compilecache(pkgid) -end diff --git a/pkgimage.mk b/pkgimage.mk new file mode 100644 index 0000000000000..aa6e6d1c266ce --- /dev/null +++ b/pkgimage.mk @@ -0,0 +1,122 @@ +SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +BUILDDIR := . +JULIAHOME := $(SRCDIR) +include $(JULIAHOME)/Make.inc + +VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) + +JULIA_DEPOT_PATH := $(build_prefix)/share/julia + +$(JULIA_DEPOT_PATH): + mkdir -p $@ + +STDLIBS := ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \ + GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll MbedTLS_jll OpenLibm_jll PCRE2_jll \ + Zlib_jll dSFMT_jll libLLVM_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar \ + LibSSH2_jll MPFR_jll LinearAlgebra Dates Distributed Future LibGit2 Profile SparseArrays UUIDs \ + SharedArrays TOML Test LibCURL Downloads Pkg Dates LazyArtifacts Sockets Unicode Markdown \ + InteractiveUtils REPL DelimitedFiles + +all-release: $(addprefix cache-release-, $(STDLIBS)) +all-debug: $(addprefix cache-debug-, $(STDLIBS)) + +define pkgimg_builder +$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ + $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) +$$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') + touch $$@ +cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image +$$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') + @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') +cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image +.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image +endef + +# Used to just define them in the dependency graph +# reside in the system image +define sysimg_builder +$$(BUILDDIR)/stdlib/$1.release.image: + touch $$@ +cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image +$$(BUILDDIR)/stdlib/$1.debug.image: + touch $$@ +cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image +.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image +endef + +# no dependencies +$(eval $(call pkgimg_builder,MozillaCACerts_jll,)) +$(eval $(call sysimg_builder,ArgTools,)) +$(eval $(call sysimg_builder,Artifacts,)) +$(eval $(call sysimg_builder,Base64,)) +$(eval $(call sysimg_builder,CRC32c,)) +$(eval $(call sysimg_builder,FileWatching,)) +$(eval $(call sysimg_builder,Libdl,)) +$(eval $(call sysimg_builder,Logging,)) +$(eval $(call sysimg_builder,Mmap,)) +$(eval $(call sysimg_builder,NetworkOptions,)) +$(eval $(call sysimg_builder,SHA,)) +$(eval $(call sysimg_builder,Serialization,)) +$(eval $(call sysimg_builder,Sockets,)) +$(eval $(call sysimg_builder,Unicode,)) + +# 1-depth packages +$(eval $(call pkgimg_builder,GMP_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,LLVMLibUnwind_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,LibUV_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,LibUnwind_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,MbedTLS_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,nghttp2_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,OpenLibm_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,PCRE2_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,Zlib_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,dSFMT_jll,Artifacts Libdl)) +$(eval $(call pkgimg_builder,libLLVM_jll,Artifacts Libdl)) +$(eval $(call sysimg_builder,libblastrampoline_jll,Artifacts Libdl)) +$(eval $(call sysimg_builder,OpenBLAS_jll,Artifacts Libdl)) +$(eval $(call sysimg_builder,Markdown,Base64)) +$(eval $(call sysimg_builder,Printf,Unicode)) +$(eval $(call sysimg_builder,Random,Serialization SHA)) +$(eval $(call sysimg_builder,Tar,ArgTools,SHA)) +$(eval $(call pkgimg_builder,DelimitedFiles,Mmap)) + +# 2-depth packages +$(eval $(call pkgimg_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl)) +$(eval $(call pkgimg_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) +$(eval $(call pkgimg_builder,MPFR_jll,Artifacts Libdl GMP_jll)) +$(eval $(call sysimg_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll)) +$(eval $(call sysimg_builder,Dates,Printf)) +$(eval $(call sysimg_builder,Distributed,Random Serialization Sockets)) +$(eval $(call sysimg_builder,Future,Random)) +$(eval $(call sysimg_builder,InteractiveUtils,Markdown)) +$(eval $(call sysimg_builder,LibGit2,NetworkOptions Printf SHA Base64)) +$(eval $(call sysimg_builder,Profile,Printf)) +$(eval $(call sysimg_builder,UUIDs,Random SHA)) + + + # 3-depth packages + # LibGit2_jll +$(eval $(call pkgimg_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl)) +$(eval $(call sysimg_builder,REPL,InteractiveUtils Markdown Sockets Unicode)) +$(eval $(call sysimg_builder,SharedArrays,Distributed Mmap Random Serialization)) +$(eval $(call sysimg_builder,TOML,Dates)) +$(eval $(call sysimg_builder,Test,Logging Random Serialization InteractiveUtils)) + +# 4-depth packages +$(eval $(call sysimg_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) + +# 5-depth packages +$(eval $(call sysimg_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) + +# 6-depth packages +$(eval $(call sysimg_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL + +# 7-depth packages +$(eval $(call sysimg_builder,LazyArtifacts,Artifacts Pkg)) + +$(eval $(call pkgimg_builder,SparseArrays,Libdl LinearAlgebra Random Serialization)) +# SuiteSparse_jll +# Statistics diff --git a/stdlib/.gitignore b/stdlib/.gitignore index d159427c40d7c..dec1745520d4c 100644 --- a/stdlib/.gitignore +++ b/stdlib/.gitignore @@ -23,3 +23,4 @@ /SHA /*_jll/StdlibArtifacts.toml /*/Manifest.toml +/*.image diff --git a/stdlib/LLD_jll/Project.toml b/stdlib/LLD_jll/Project.toml index 923f2e578e0d7..52d4b67de3765 100644 --- a/stdlib/LLD_jll/Project.toml +++ b/stdlib/LLD_jll/Project.toml @@ -3,10 +3,8 @@ uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" version = "14.0.6+3" [deps] -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" libLLVM_jll = "8f36deef-c2a5-5394-99ed-8e07531fb29a" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"