Support for DPKG local caching

Makefile.work

@@ -30,6 +30,8 @@
 #  *                 Default: yes
 #  *                 Values:  yes, no
 #  * KERNEL_PROCURE_METHOD: Specifying method of obtaining kernel Debian package: download or build
+#  * SONIC_DPKG_CACHE_METHOD: Specifying method of obtaining the Debian packages from cache: none or cache
+#  * SONIC_DPKG_CACHE_SOURCE: Debian package cache location when cache enabled for debian packages
 #
 ###############################################################################
 
@@ -137,6 +139,12 @@ ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
 
 endif
 
+ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
+    $(shell test -d $(SONIC_DPKG_CACHE_SOURCE) || mkdir -p $(SONIC_DPKG_CACHE_SOURCE) )
+    DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache"
+endif
+
+
 DOCKER_BASE_BUILD = docker build --no-cache \
 		    -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
 		    --build-arg http_proxy=$(http_proxy) \
@@ -171,6 +179,8 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_USE_DOCKER_BUILDKIT=$(SONIC_USE_DOCKER_BUILDKIT) \
                            VS_PREPARE_MEM=$(VS_PREPARE_MEM) \
                            KERNEL_PROCURE_METHOD=$(KERNEL_PROCURE_METHOD) \
+                           SONIC_DPKG_CACHE_METHOD=$(SONIC_DPKG_CACHE_METHOD) \
+                           SONIC_DPKG_CACHE_SOURCE=$(SONIC_DPKG_CACHE_SOURCE) \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
                            SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \

rules/config

@@ -93,5 +93,17 @@ DEFAULT_KERNEL_PROCURE_METHOD = build
 FRR_USER_UID = 300
 FRR_USER_GID = 300
 
+# DPKG cache allows the .deb files to be stored in the cache path. This allows the submodules
+# package to be cached and restored back if its commit hash is not modified and its dependencies are not modified.
+# SONIC_DPKG_CACHE_METHOD - Default method of deb package caching
+#	none:     no caching
+#	cache:    cache from local directory
+#	#url:      cache over remote http server (not supported)
+#	#registry: cache on the docker local/remote registry (not supported)
+# SONIC_DPKG_CACHE_SOURCE - provides the cache location detail
+SONIC_DPKG_CACHE_METHOD=cache
+SONIC_DPKG_CACHE_SOURCE=${PWD}/target/cache/
+
+
 # Default VS build memory preparation
 DEFAULT_VS_PREPARE_MEM = yes

rules/linux-kernel.mk

@@ -7,8 +7,19 @@ KERNEL_SUBVERSION = 1+deb9u3
 
 export KVERSION_SHORT KVERSION KERNEL_VERSION KERNEL_SUBVERSION
 
+
+SMPATH=$(SRC_PATH)/sonic-linux-kernel/
+SM_DEP_LIST  := Makefile
+SM_DEP_LIST  += patch/*
+SM_DEP_LIST  += patch/preconfig/*
+SMDEP_LIST   := $(wildcard $(addprefix $(SMPATH),$(SM_DEP_LIST)))
+DEP_LIST     := $(SONIC_MAKEFILE_LIST) rules/linux-kernel.mk
+
 LINUX_HEADERS_COMMON = linux-headers-$(KVERSION_SHORT)-common_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_all.deb
 $(LINUX_HEADERS_COMMON)_SRC_PATH = $(SRC_PATH)/sonic-linux-kernel
+$(LINUX_HEADERS_COMMON)_CACHE_MODE = GIT_COMMIT_SHA
+$(LINUX_HEADERS_COMMON)_DEP_SOURCE = $(DEP_LIST)
+$(LINUX_HEADERS_COMMON)_SMDEP_SOURCE = $(SMDEP_LIST)
 SONIC_MAKE_DEBS += $(LINUX_HEADERS_COMMON)
 
 LINUX_HEADERS = linux-headers-$(KVERSION)_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_$(CONFIGURED_ARCH).deb

slave.mk

@@ -48,6 +48,7 @@ export BUILD_NUMBER
 export BUILD_TIMESTAMP
 export CONFIGURED_PLATFORM
 export CONFIGURED_ARCH
+SONIC_MAKEFILE_LIST=slave.mk rules/config rules/functions
 
 ###############################################################################
 ## Utility rules
@@ -180,6 +181,9 @@ $(info "ENABLE_SYSTEM_TELEMETRY"         : "$(ENABLE_SYSTEM_TELEMETRY)")
 $(info "SONIC_DEBUGGING_ON"              : "$(SONIC_DEBUGGING_ON)")
 $(info "SONIC_PROFILING_ON"              : "$(SONIC_PROFILING_ON)")
 $(info "KERNEL_PROCURE_METHOD"           : "$(KERNEL_PROCURE_METHOD)")
+ifeq ($(SONIC_DPKG_CACHE_METHOD),cache)
+$(info "DPKG_CACHE_PATH"                 : "$(SONIC_DPKG_CACHE_SOURCE)")
+endif
 $(info "BUILD_TIMESTAMP"                 : "$(BUILD_TIMESTAMP)")
 $(info "BLDENV"                          : "$(BLDENV)")
 $(info "VS_PREPARE_MEM"                  : "$(VS_PREPARE_MEM)")
@@ -200,6 +204,85 @@ export kernel_procure_method=$(KERNEL_PROCURE_METHOD)
 export vs_build_prepare_mem=$(VS_PREPARE_MEM)
 
 ###############################################################################
+MOD_CACHE_LOCK_TIMEOUT = 3600
+SONIC_DPKG_CACHE_DIR=/dpkg_cache
+
+# Lock macro for debian package level cache
+# Lock is implemented through flock command with the timeout value of 1 hour
+# Lock file is created in the cache directory and corresponding lock fd is stored as part of DPKG recipe.
+define MOD_LOCK
+	if [[ ! -f $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock ]]; then
+		touch $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock
+		chmod 777 $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock;
+	fi
+	$(eval $(1)_lock_fd=$(subst -,_,$(subst +,_,$(subst .,_,$(1)))))
+	exec {$($(1)_lock_fd)}<"$(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock";
+	if ! flock -x -w $(MOD_CACHE_LOCK_TIMEOUT) "$${$($(1)_lock_fd)}" ; then
+		echo "ERROR: Lock timeout trying to read $(1) from cache.";
+		exit 1;
+	fi
+endef
+
+
+# UnLock macro for debian package level cache
+define MOD_UNLOCK
+	eval exec "$${$($(1)_lock_fd)}<&-";
+endef
+
+
+# Loads the deb package from debian cache
+# Cache file prefix is formed using SHA value
+# The SHA value is derived from one of the keyword type - GIT_COMMIT_SHA or GIT_CONTENT_SHA
+#   GIT_COMMIT_SHA   - SHA value of the last git commit id if it is a submodule
+#   GIT_CONTENT_SHA  - SHA value is calculated from the target dependency files content.
+#   Cache is loaded only when corresponding cache file is present in the cache direcory and its dependencies are not changed.
+define LOAD_CACHE
+	$(eval MOD_SRC_PATH=$($(1)_SRC_PATH))
+	$(eval MOD_HASH=$(if $(filter GIT_COMMIT_SHA,$($(1)_CACHE_MODE)),$(shell cd $(MOD_SRC_PATH) && git log -1 --format="%H")
+		, $(shell git hash-object $($(1)_DEP_SOURCE) $($(1)_SMDEP_SOURCE)|sha1sum|awk '{print $$1}')))
+	$(eval MOD_CACHE_FILE=$(1)-$(MOD_HASH).tgz)
+	$(eval $(1)_MOD_CACHE_FILE=$(MOD_CACHE_FILE))
+	$(eval DRV_DEB=$(foreach pkg,$(addprefix $(DEBS_PATH)/,$(1) $($(1)_DERIVED_DEBS)),$(if $(wildcard $(pkg)),,$(pkg))))
+	$(eval $(1)_FILES_MODIFIED  := $(if $($(1)_DEP_SOURCE),$(shell git status -s $($(1)_DEP_SOURCE))) \
+		   $(if $($(1)_SMDEP_SOURCE),$(shell cd $(MOD_SRC_PATH) &&  git status -s $(subst $(MOD_SRC_PATH)/,,$($(1)_SMDEP_SOURCE)))) )
+	#$(filter-out $($(1)_DEP_SOURCE),$($(1)_SMDEP_SOURCE), $?)
+
+	$(if $($(1)_FILES_MODIFIED),
+		echo "Target $(1) dependencies are modifed - load cache skipped";
+		echo "Modified dependencies are : [$($(1)_FILES_MODIFIED)] ";
+	    ,
+		$(if $(wildcard $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE)),
+			$(if $(DRV_DEB), tar -xzvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE),echo );
+			echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is loaded from cache";
+			$(eval $(1)_CACHE_LOADED := Yes)
+			,
+			echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is not present in the cache !";
+		 )
+	 )
+	echo ""
+endef
+
+# Saves the deb package into debian cache
+# A single tared-zip cache is created for .deb and its derived packages in the cache direcory.
+# It saves the .deb into cache only when its dependencies are not changed
+# The cache save is protected with lock.
+# The SAVE_CACHE macro has dependecy with LOAD_CACHE macro
+# 	 The target specific variables -_SRC_PATH, _MOD_CACHE_FILE and _FILES_MODIFIED are
+# 	 derived from the LOAD_CACHE macro
+define SAVE_CACHE
+	$(eval MOD_SRC_PATH=$($(1)_SRC_PATH))
+	$(eval MOD_CACHE_FILE=$($(1)_MOD_CACHE_FILE))
+	$(call MOD_LOCK,$(1))
+	$(if $($(1)_FILES_MODIFIED),
+		echo "Target $(1) dependencies are modifed - save cache skipped";
+	    ,
+		tar -czvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) $(2) $(addprefix $(DEBS_PATH)/,$($(1)_DERIVED_DEBS));
+		echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) saved in the cache ";
+	 )
+	$(call MOD_UNLOCK,$(1))
+	echo ""
+endef
+
 ## Local targets
 ###############################################################################
 
@@ -297,8 +380,16 @@ SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES))
 #     $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name
 #     $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ...
 #     SONIC_MAKE_DEBS += $(SOME_NEW_DEB)
-$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)))
+$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
+	$$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE)
 	$(HEADER)
+
+	# Load the target deb from DPKG cache
+	$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) )
+
+	# Skip building the target if it is already loaded from cache
+	if [ -z '$($*_CACHE_LOADED)' ] ; then
+
 	# Remove target to force rebuild
 	rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS))
 	# Apply series of patches if exist
@@ -307,6 +398,12 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
 	DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG)
 	# Clean up
 	if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi
+
+		# Save the target deb into DPKG cache
+		$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@))
+
+	fi
+
 	$(FOOTER)
 
 SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS))
@@ -317,8 +414,16 @@ SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS))
 #     $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name
 #     $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ...
 #     SONIC_DPKG_DEBS += $(SOME_NEW_DEB)
-$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)))
+$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
+	$$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE)
 	$(HEADER)
+
+	# Load the target deb from DPKG cache
+	$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) )
+
+	# Skip building the target if it is already loaded from cache
+	if [ -z '$($*_CACHE_LOADED)' ] ; then
+
 	# Remove old build logs if they exist
 	rm -f $($*_SRC_PATH)/debian/*.debhelper.log
 	# Apply series of patches if exist
@@ -335,6 +440,11 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
 	if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi
 	# Take built package(s)
 	mv $(addprefix $($*_SRC_PATH)/../, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) $(DEBS_PATH) $(LOG)
+
+		# Save the target deb into DPKG cache
+		$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@))
+	fi
+
 	$(FOOTER)
 
 SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS))