[arm] support compile sonic arm image on arm server (sonic-net#7285)

- Support compile sonic arm image on arm server. If arm image compiling is executed on arm server instead of using qemu mode on x86 server, compile time can be saved significantly. - Add kernel argument systemd.unified_cgroup_hierarchy=0 for upgrade systemd to version 247, according to sonic-net#7228 - rename multiarch docker to sonic-slave-${distro}-march-${arch} Co-authored-by: Xianghong Gu <xgu@centecnetworks.com> Co-authored-by: Shi Lei <shil@centecnetworks.com>
raphaelt-nvidia · May 13, 2021 · babc825 · babc825
1 parent e37ff79
commit babc825
Show file tree

Hide file tree

Showing 18 changed files with 66 additions and 38 deletions.
diff --git a/Makefile.work b/Makefile.work
@@ -49,6 +49,15 @@ USER_LC := $(shell echo $(USER) | tr A-Z a-z)
 
 comma := ,
 
+ifeq ($(shell uname -m), aarch64)
+COMPILE_HOST_ARCH := arm64
+else ifeq ($(shell uname -m), armv7l)
+COMPILE_HOST_ARCH := armhf
+else
+COMPILE_HOST_ARCH := amd64
+endif
+
+
 ifeq ($(USER), root)
 $(error Add your user account to docker group and use your user account to make. root or sudo are not supported!)
 endif
@@ -99,8 +108,15 @@ endif
 
 ifeq ($(CONFIGURED_ARCH),amd64)
 SLAVE_BASE_IMAGE = $(SLAVE_DIR)
+MULTIARCH_QEMU_ENVIRON = n
 else
-SLAVE_BASE_IMAGE = $(SLAVE_DIR)-$(CONFIGURED_ARCH)
+ifeq ($(CONFIGURED_ARCH), $(COMPILE_HOST_ARCH))
+SLAVE_BASE_IMAGE = $(SLAVE_DIR)
+MULTIARCH_QEMU_ENVIRON = n
+else ifneq ($(CONFIGURED_ARCH),)
+SLAVE_BASE_IMAGE = $(SLAVE_DIR)-march-$(CONFIGURED_ARCH)
+MULTIARCH_QEMU_ENVIRON = y
+endif
 endif
 SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
 
@@ -110,8 +126,8 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
 $(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
@@ -159,7 +175,7 @@ ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
 endif
 
-ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 ifeq ($(DOCKER_DATA_ROOT_FOR_MULTIARCH),)
 	DOCKER_DATA_ROOT_FOR_MULTIARCH := /var/lib/march/docker
 endif
@@ -219,6 +235,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            -f slave.mk \
                            PLATFORM=$(PLATFORM) \
                            PLATFORM_ARCH=$(PLATFORM_ARCH) \
+                           MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON)  \
                            BUILD_NUMBER=$(BUILD_NUMBER) \
                            BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) \
                            SONIC_IMAGE_VERSION=$(SONIC_IMAGE_VERSION) \

diff --git a/build_debian.sh b/build_debian.sh
@@ -189,6 +189,9 @@ if [ -f platform/$CONFIGURED_PLATFORM/modules ]; then
     cat platform/$CONFIGURED_PLATFORM/modules | sudo tee -a $FILESYSTEM_ROOT/etc/initramfs-tools/modules > /dev/null
 fi
 
+## Add mtd and uboot firmware tools package
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools mtd-utils device-tree-compiler
+
 ## Install docker
 echo '[INFO] Install docker'
 ## Install apparmor utils since they're missing and apparmor is enabled in the kernel
@@ -495,7 +498,7 @@ fi
 sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum
 
 if [ -f sonic_debian_extension.sh ]; then
-    ./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR
+    ./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR $IMAGE_DISTRO
 fi
 
 ## Organization specific extensions such as Configuration & Scripts for features like AAA, ZTP...
@@ -528,9 +531,6 @@ then
 
 fi
 
-## Add mtd and uboot firmware tools package
-sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools mtd-utils device-tree-compiler
-
 ## Update initramfs
 sudo chroot $FILESYSTEM_ROOT update-initramfs -u
 ## Convert initrd image to u-boot format
@@ -582,7 +582,7 @@ sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker
 
 scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
 
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [ $MULTIARCH_QEMU_ENVIRON == y ]; then
     # Remove qemu arm bin executable used for cross-building
     sudo rm -f $FILESYSTEM_ROOT/usr/bin/qemu*static || true
     DOCKERFS_PATH=../dockerfs/

diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2
@@ -1,7 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-{% if CONFIGURED_ARCH == "armhf" %}
+{% if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:armhf-buster
-{% elif CONFIGURED_ARCH == "arm64" %}
+{% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-buster
 {% else %}
 FROM debian:buster

diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2
@@ -1,7 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-{% if CONFIGURED_ARCH == "armhf" %}
+{% if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:armhf-stretch
-{% elif CONFIGURED_ARCH == "arm64" %}
+{% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-stretch
 {% else %}
 FROM debian:stretch

diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2
@@ -1,6 +1,6 @@
-{% if CONFIGURED_ARCH == "armhf" %}
+{% if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:armhf-jessie
-{% elif CONFIGURED_ARCH == "arm64" %}
+{% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-jessie
 {% else %}
 FROM debian:jessie

diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2
@@ -1,6 +1,6 @@
-{% if CONFIGURED_ARCH == "armhf" %}
+{% if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:armhf-stretch
-{% elif CONFIGURED_ARCH == "arm64" %}
+{% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-stretch
 {% else %}
 FROM debian:stretch

diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2
@@ -21,6 +21,12 @@ PLATFORM_DIR=$2
     exit 1
 }
 
+IMAGE_DISTRO=$3
+[ -n "$IMAGE_DISTRO" ] || {
+    echo "Error: no or empty IMAGE_DISTRO argument"
+    exit 1
+}
+
 ## Enable debug output for script
 set -x -e
 
@@ -61,7 +67,7 @@ sudo bash -c "echo \"DOCKER_OPTS=\"--storage-driver=overlay2\"\" >> $FILESYSTEM_
 # Copy docker start script to be able to start docker in chroot
 sudo mkdir -p "$FILESYSTEM_ROOT/$DOCKER_CTL_DIR"
 sudo cp $DOCKER_SCRIPTS_DIR/docker "$FILESYSTEM_ROOT/$DOCKER_CTL_SCRIPT"
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [ $MULTIARCH_QEMU_ENVIRON == y ]; then
     SONIC_NATIVE_DOCKERD_FOR_DOCKERFS=" -H unix:///dockerfs/var/run/docker.sock "
     SONIC_NATIVE_DOCKERD_FOR_DOCKERFS_PID="cat `pwd`/dockerfs/var/run/docker.pid"
 else
@@ -585,7 +591,7 @@ EOF
 {% endif %}
 
 ## Bind docker path
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [ $MULTIARCH_QEMU_ENVIRON == y ]; then
     sudo mkdir -p $FILESYSTEM_ROOT/dockerfs
     sudo mount --bind dockerfs $FILESYSTEM_ROOT/dockerfs
 fi
@@ -623,7 +629,7 @@ echo "docker images pull complete"
 
 sudo umount $FILESYSTEM_ROOT/target
 sudo rm -r $FILESYSTEM_ROOT/target
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
+if [ $MULTIARCH_QEMU_ENVIRON == y ]; then
     sudo umount $FILESYSTEM_ROOT/dockerfs
     sudo rm -fr $FILESYSTEM_ROOT/dockerfs
     sudo kill -9 `sudo $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS_PID` || true

diff --git a/platform/centec-arm64/platform.conf b/platform/centec-arm64/platform.conf
@@ -37,7 +37,7 @@ bootloader_menu_config() {
     (cat <<EOF
 hw_load $hw_load_str
 copy_img echo "Loading Demo $platform image..." && run hw_load
-nos_bootcmd run copy_img && setenv bootargs quiet console=\$consoledev,\$baudrate root=/dev/mmcblk0p1 rw rootwait rootfstype=ext4 loopfstype=squashfs loop=$image_dir/fs.squashfs && bootm \$loadaddr
+nos_bootcmd run copy_img && setenv bootargs quiet console=\$consoledev,\$baudrate root=/dev/mmcblk0p1 rw rootwait rootfstype=ext4 loopfstype=squashfs loop=$image_dir/fs.squashfs systemd.unified_cgroup_hierarchy=0 && bootm \$loadaddr
 EOF
     ) > /tmp/env.txt
 

diff --git a/platform/marvell-arm64/platform.conf b/platform/marvell-arm64/platform.conf
@@ -108,8 +108,8 @@ prepare_boot_menu() {
     BORDER='echo "---------------------------------------------------";echo;'
     fw_setenv ${FW_ARG} print_menu $BORDER $BOOT1 $BOOT2 $BOOT3 $BORDER > /dev/null
 
-    fw_setenv ${FW_ARG} linuxargs "net.ifnames=0 loopfstype=squashfs loop=$image_dir/$FILESYSTEM_SQUASHFS varlog_size=$VAR_LOG" > /dev/null
-    fw_setenv ${FW_ARG} linuxargs_old "net.ifnames=0 loopfstype=squashfs loop=$image_dir_old/$FILESYSTEM_SQUASHFS varlog_size=$VAR_LOG" > /dev/null
+    fw_setenv ${FW_ARG} linuxargs "net.ifnames=0 loopfstype=squashfs loop=$image_dir/$FILESYSTEM_SQUASHFS systemd.unified_cgroup_hierarchy=0 varlog_size=$VAR_LOG" > /dev/null
+    fw_setenv ${FW_ARG} linuxargs_old "net.ifnames=0 loopfstype=squashfs loop=$image_dir_old/$FILESYSTEM_SQUASHFS systemd.unified_cgroup_hierarchy=0 varlog_size=$VAR_LOG" > /dev/null
     sonic_bootargs_old='setenv bootargs root='$demo_dev' rw rootwait rootfstype=ext4 panic=1 console=ttyS0,115200 ${othbootargs} ${mtdparts} ${linuxargs_old}'
     fw_setenv ${FW_ARG} sonic_bootargs_old $sonic_bootargs_old > /dev/null || true
     sonic_boot_load_old=$(fw_printenv -n sonic_boot_load || true)

diff --git a/platform/marvell-armhf/platform.conf b/platform/marvell-armhf/platform.conf
@@ -146,8 +146,8 @@ prepare_boot_menu() {
     BORDER='echo "---------------------------------------------------";echo;'
     fw_setenv ${FW_ARG} print_menu $BORDER $BOOT1 $BOOT2 $BOOT3 $BORDER > /dev/null
 
-    fw_setenv ${FW_ARG} linuxargs "net.ifnames=0 loopfstype=squashfs loop=$image_dir/$FILESYSTEM_SQUASHFS varlog_size=$VAR_LOG loglevel=4" > /dev/null
-    fw_setenv ${FW_ARG} linuxargs_old "net.ifnames=0 loopfstype=squashfs loop=$image_dir_old/$FILESYSTEM_SQUASHFS varlog_size=$VAR_LOG loglevel=4" > /dev/null
+    fw_setenv ${FW_ARG} linuxargs "net.ifnames=0 loopfstype=squashfs loop=$image_dir/$FILESYSTEM_SQUASHFS systemd.unified_cgroup_hierarchy=0 varlog_size=$VAR_LOG loglevel=4" > /dev/null
+    fw_setenv ${FW_ARG} linuxargs_old "net.ifnames=0 loopfstype=squashfs loop=$image_dir_old/$FILESYSTEM_SQUASHFS systemd.unified_cgroup_hierarchy=0 varlog_size=$VAR_LOG loglevel=4" > /dev/null
 
     # Set boot configs
     fw_setenv ${FW_ARG} kernel_addr $kernel_addr > /dev/null

diff --git a/scripts/build_debian_base_system.sh b/scripts/build_debian_base_system.sh
@@ -20,9 +20,11 @@ generate_version_file()
 
 if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then
     if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
-        # qemu arm bin executable for cross-building
-        sudo mkdir -p $FILESYSTEM_ROOT/usr/bin
-        sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true
+        if [ $MULTIARCH_QEMU_ENVIRON == y ]; then
+            # qemu arm bin executable for cross-building
+            sudo mkdir -p $FILESYSTEM_ROOT/usr/bin
+            sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true
+        fi
         sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian
     else
         sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian

diff --git a/slave.mk b/slave.mk
@@ -61,6 +61,7 @@ export CONFIGURED_ARCH
 export PYTHON_WHEELS_PATH
 export IMAGE_DISTRO
 export IMAGE_DISTRO_DEBS_PATH
+export MULTIARCH_QEMU_ENVIRON
 
 ###############################################################################
 ## Utility rules
@@ -259,6 +260,7 @@ $(info "INCLUDE_KUBERNETES"              : "$(INCLUDE_KUBERNETES)")
 $(info "INCLUDE_MACSEC"                  : "$(INCLUDE_MACSEC)")
 $(info "TELEMETRY_WRITABLE"              : "$(TELEMETRY_WRITABLE)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
+$(info "MULTIARCH_QEMU_ENVIRON"          : "$(MULTIARCH_QEMU_ENVIRON)")
 $(info )
 else
 $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))
@@ -1016,6 +1018,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+	MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
 		./build_debian.sh $(LOG)
 
 	USERNAME="$(USERNAME)" \

diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
@@ -1,8 +1,8 @@
-{%- if CONFIGURED_ARCH == "armhf" %}
+{%- if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/qemu-user-static:x86_64-arm-5.0.0-2 as qemu
 FROM multiarch/debian-debootstrap:armhf-buster
 COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
-{%- elif CONFIGURED_ARCH == "arm64" %}
+{%- elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-buster
 {%- else -%}
 FROM debian:buster

diff --git a/sonic-slave-buster/Dockerfile.user.j2 b/sonic-slave-buster/Dockerfile.user.j2
@@ -1,8 +1,8 @@
 ARG slave_base_tag_ref=latest
-{%- if CONFIGURED_ARCH == "amd64" %}
+{%- if MULTIARCH_QEMU_ENVIRON != "y" %}
 FROM sonic-slave-buster:${slave_base_tag_ref}
 {%- else %}
-FROM sonic-slave-buster-{{ CONFIGURED_ARCH }}:${slave_base_tag_ref}
+FROM sonic-slave-buster-march-{{ CONFIGURED_ARCH }}:${slave_base_tag_ref}
 {%- endif %}
 
 # Add user

diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2
@@ -1,6 +1,6 @@
-{%- if CONFIGURED_ARCH == "armhf" %}
+{%- if CONFIGURED_ARCH == "armhf" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:armhf-stretch
-{%- elif CONFIGURED_ARCH == "arm64" %}
+{%- elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM multiarch/debian-debootstrap:arm64-stretch
 {%- else -%}
 FROM debian:stretch

diff --git a/sonic-slave-stretch/Dockerfile.user.j2 b/sonic-slave-stretch/Dockerfile.user.j2
@@ -1,8 +1,8 @@
 ARG slave_base_tag_ref=latest
-{%- if CONFIGURED_ARCH == "amd64"  %}
+{%- if MULTIARCH_QEMU_ENVIRON != "y"  %}
 FROM sonic-slave-stretch:${slave_base_tag_ref}
 {%- else %}
-FROM sonic-slave-stretch-{{ CONFIGURED_ARCH }}:${slave_base_tag_ref}
+FROM sonic-slave-stretch-march-{{ CONFIGURED_ARCH }}:${slave_base_tag_ref}
 {%- endif %}
 
 # Add user

diff --git a/src/ethtool/Makefile b/src/ethtool/Makefile
@@ -12,7 +12,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
 	pushd ethtool
 	git checkout tags/debian/1%$(ETHTOOL_VERSION_BASE)-1
 	# Build package
-ifneq (,(filter $(CONFIGURED_ARCH),arm64 armhf))
+ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 	DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
 else
 	dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)

diff --git a/src/snmpd/Makefile b/src/snmpd/Makefile
@@ -29,7 +29,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
 	stg init
 	stg import -s ../patch-$(SNMPD_VERSION)/series
 
-ifeq ($(CONFIGURED_ARCH), arm64)
+ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 	dpkg-buildpackage -rfakeroot -b -d -us -uc -j1 --admindir $(SONIC_DPKG_ADMINDIR)
 else
 	dpkg-buildpackage -rfakeroot -b -d -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)