Add netconf docker to build-image

update

Signed-off-by: Hossam Sherif <hossamsherifmostafa@gmail.com>

.gitmodules

@@ -124,3 +124,6 @@
 [submodule "platform/innovium/sonic-platform-marvell"]
 	path = platform/innovium/sonic-platform-marvell
 	url = https://github.com/Marvell-switching/sonic-platform-marvell
+[submodule "src/sonic-netconf-server"]
+	path = src/sonic-netconf-server
+	url = https://github.com/sonic-net/sonic-netconf-server

README.md

@@ -414,3 +414,4 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
 For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
 or contact [opencode@microsoft.com](mailto:opencode@microsoft.com)
 with any additional questions or comments.
+

dockers/docker-sonic-netconf-server/Dockerfile.j2

@@ -0,0 +1,53 @@
+FROM docker-config-engine-buster
+
+ARG docker_container_name
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y g++ python python2 python3-dev libxml2 libcurl3-gnutls libcjson-dev
+
+# TODO: Remove these lines once we no longer need Python 2
+RUN apt-get install -f -y python-dev python-pip
+RUN pip2 install --upgrade 'pip<21'
+RUN apt-get purge -y python-pip
+RUN pip2 install setuptools==40.8.0
+RUN pip2 install wheel==0.35.1
+RUN pip2 install connexion==1.1.15 \
+                setuptools==21.0.0 \
+                grpcio-tools==1.20.0 \
+                certifi==2017.4.17 \
+                python-dateutil==2.6.0 \
+                six==1.11.0 \
+                urllib3==1.21.1
+
+RUN pip3 install connexion==2.7.0 \
+                 setuptools==21.0.0 \
+                 grpcio-tools==1.20.0 \
+                 certifi==2017.4.17 \
+                 python-dateutil==2.6.0 \
+                 six==1.11.0 \
+                 urllib3==1.26.5
+
+COPY \
+{% for deb in docker_sonic_netconf_server_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor -%}
+debs/
+
+RUN dpkg -i \
+{% for deb in docker_sonic_netconf_server_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor %}
+
+COPY ["start.sh", "netconf-server.sh", "/usr/bin/"]
+COPY ["mgmt_vars.j2", "/usr/share/sonic/templates/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+
+RUN apt-get remove -y g++ python3-dev
+RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN rm -rf /debs
+
+ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-sonic-netconf-server/mgmt_vars.j2

@@ -0,0 +1,5 @@
+{
+    "netconf_server": {% if NETCONF_SERVER is defined and "default" in NETCONF_SERVER.keys() %}{{ NETCONF_SERVER['default'] }}{% else %}""{% endif %},
+    "x509" : {% if "x509" in DEVICE_METADATA.keys() %}{{ DEVICE_METADATA["x509"] }}{% else %}""{% endif %},
+    "tacacs_plus" :  {%if TACPLUS_SERVER is defined  %}{{ "true" }}{% else %}""{% endif %}
+}

dockers/docker-sonic-netconf-server/netconf-server.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Startup script for SONiC Management NETCONF Server
+EXIT_MGMT_VARS_FILE_NOT_FOUND=1
+MGMT_VARS_FILE=/usr/share/sonic/templates/mgmt_vars.j2
+
+if [ ! -f "$MGMT_VARS_FILE" ]; then
+    echo "Mgmt vars template file not found"
+    exit $EXIT_MGMT_VARS_FILE_NOT_FOUND
+fi
+
+# Read basic server settings from mgmt vars entries
+MGMT_VARS=$(sonic-cfggen -d -t $MGMT_VARS_FILE)
+MGMT_VARS=${MGMT_VARS//[\']/\"}
+
+NETCONF_SERVER=$(echo $MGMT_VARS | jq -r '.netconf_server')
+
+if [ -n "$NETCONF_SERVER" ]; then
+    SERVER_PORT=$(echo $NETCONF_SERVER | jq -r '.port')
+    LOG_LEVEL=$(echo $NETCONF_SERVER | jq -r '.log_level')
+else
+    SERVER_PORT=830
+    LOG_LEVEL=5
+fi
+
+NETCONF_SERVER_ARGS="-logtostderr"
+[ ! -z $SERVER_PORT ] && NETCONF_SERVER_ARGS+=" -port $SERVER_PORT"
+[ ! -z $LOG_LEVEL   ] && NETCONF_SERVER_ARGS+=" -v $LOG_LEVEL"
+
+echo "NETCONF_SERVER_ARGS = $NETCONF_SERVER_ARGS"
+
+export CVL_SCHEMA_PATH=/usr/sbin/schema
+
+exec /usr/sbin/netconf_server ${NETCONF_SERVER_ARGS}

dockers/docker-sonic-netconf-server/start.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+mkdir -p /var/sonic
+echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status

dockers/docker-sonic-netconf-server/supervisord.conf

@@ -0,0 +1,44 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=1024
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:start]
+command=/usr/bin/start.sh
+priority=2
+autostart=true
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+
+[program:netconf-server]
+command=/usr/bin/netconf-server.sh
+priority=4
+autostart=false
+autorestart=true
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=start:exited
+

files/build_templates/init_cfg.json.j2

@@ -49,6 +49,7 @@
 {%- if sonic_asic_platform == "vs" %}{% do features.append(("gbsyncd", "enabled", false, "enabled")) %}{% endif %}
 {%- if include_iccpd == "y" %}{% do features.append(("iccpd", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_mgmt_framework == "y" %}{% do features.append(("mgmt-framework", "enabled", true, "enabled")) %}{% endif %}
+{%- if include_netconf_server == "y" %}{% do features.append(("netconf-server", "enabled", true, "enabled")) %}{% endif %}
 {%- if include_mux == "y" %}{% do features.append(("mux", "{% if 'subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR' %}enabled{% else %}always_disabled{% endif %}", false, "enabled")) %}{% endif %}
 {%- if include_nat == "y" %}{% do features.append(("nat", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_p4rt == "y" %}{% do features.append(("p4rt", "disabled", false, "enabled")) %}{% endif %}

files/build_templates/netconf-server.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Netconf Server container
+Requires=database.service
+After=database.service swss.service syncd.service
+BindsTo=sonic.target
+After=sonic.target
+Before=ntp-config.service
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop

files/build_templates/netconf-server.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Delays management framework container until SONiC has started
+PartOf=netconf-server.service
+
+[Timer]
+OnUnitActiveSec=0 sec
+OnBootSec=3min 30 sec
+Unit=netconf-server.service
+
+[Install]
+WantedBy=timers.target sonic.target sonic-delayed.target

files/build_templates/sonic_debian_extension.j2

@@ -1026,6 +1026,11 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y
 
+{% if include_netconf_server == 'y' %}
+sudo cp $BUILD_TEMPLATES/netconf-server.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
+echo "netconf-server.timer" | sudo tee -a $GENERATED_SERVICE_FILE
+{% endif %}
+
 {% for file in installer_extra_files.split(' ') -%}
 {% if file.strip() -%}
 {% set src = file.split(':')[0] -%}

rules/config

@@ -136,6 +136,9 @@ INCLUDE_SFLOW = y
 # INCLUDE_MGMT_FRAMEWORK - build docker-sonic-mgmt-framework for CLI and REST server support
 INCLUDE_MGMT_FRAMEWORK = y
 
+# INCLUDE_NETCONF_SERVER - build docker-sonic-netconf server for NETCONF support
+INCLUDE_NETCONF_SERVER = y
+
 # ENABLE_HOST_SERVICE_ON_START - enable sonic-host-server for mgmt-framework and/or
 # gnmi containers to access host functionality by default
 ENABLE_HOST_SERVICE_ON_START = y

rules/docker-sonic-netconf-server.dep

@@ -0,0 +1,11 @@
+
+DPATH       := $($(DOCKER_NETCONF_SERVER)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-sonic-netconf-server.mk rules/docker-sonic-netconf-server.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_NETCONF_SERVER)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_NETCONF_SERVER)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_NETCONF_SERVER)_DEP_FILES   := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_NETCONF_SERVER),$(DOCKER_NETCONF_SERVER_DBG)))

rules/docker-sonic-netconf.mk

@@ -0,0 +1,39 @@
+# docker image for netconf-server
+
+DOCKER_NETCONF_SERVER_STEM = docker-sonic-netconf-server
+DOCKER_NETCONF_SERVER = $(DOCKER_NETCONF_SERVER_STEM).gz
+DOCKER_NETCONF_SERVER_DBG = $(DOCKER_NETCONF_SERVER_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_NETCONF_SERVER)_PATH = $(DOCKERS_PATH)/$(DOCKER_NETCONF_SERVER_STEM)
+
+$(DOCKER_NETCONF_SERVER)_DEPENDS += $(SONIC_MGMT_COMMON)
+$(DOCKER_NETCONF_SERVER)_DEPENDS += $(SONIC_NETCONF_SERVER)
+$(DOCKER_NETCONF_SERVER)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS)
+$(DOCKER_NETCONF_SERVER)_DBG_DEPENDS += $(SONIC_NETCONF_SERVER_DBG)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_NETCONF_SERVER)
+$(DOCKER_NETCONF_SERVER)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
+
+$(DOCKER_NETCONF_SERVER)_VERSION = 1.0.0
+$(DOCKER_NETCONF_SERVER)_PACKAGE_NAME = netconf-server
+
+$(DOCKER_NETCONF_SERVER)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
+
+ifeq ($(INCLUDE_NETCONF_SERVER), y)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_NETCONF_SERVER)
+endif
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_NETCONF_SERVER_DBG)
+ifeq ($(INCLUDE_NETCONF_SERVER), y)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_NETCONF_SERVER_DBG)
+endif
+
+$(DOCKER_NETCONF_SERVER)_CONTAINER_NAME = netconf-server
+$(DOCKER_NETCONF_SERVER)_RUN_OPT += --privileged -t
+$(DOCKER_NETCONF_SERVER)_RUN_OPT += -v /etc/sonic:/etc/sonic:rw
+$(DOCKER_NETCONF_SERVER)_RUN_OPT += -v /etc:/host_etc:ro
+$(DOCKER_NETCONF_SERVER)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
+$(DOCKER_NETCONF_SERVER)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
+
+SONIC_BUSTER_DOCKERS += $(DOCKER_NETCONF_SERVER)
+SONIC_BUSTER_DBG_DOCKERS += $(DOCKER_NETCONF_SERVER_DBG)

rules/sonic-netconf-server.dep

@@ -0,0 +1,12 @@
+
+SPATH       := $($(SONIC_NETCONF_SERVER)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-netconf-server.mk rules/sonic-netconf-server.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_NETCONF_SERVER)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(SONIC_NETCONF_SERVER)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_NETCONF_SERVER)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_NETCONF_SERVER)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_NETCONF_SERVER)_SMDEP_PATHS := $(SPATH)
+

rules/sonic-netconf-server.mk

@@ -0,0 +1,16 @@
+# SONiC netconf-server package
+
+ifeq ($(INCLUDE_NETCONF_SERVER), y)
+
+SONIC_NETCONF_SERVER = sonic-netconf-server_1.0-01_$(CONFIGURED_ARCH).deb
+$(SONIC_NETCONF_SERVER)_SRC_PATH = $(SRC_PATH)/sonic-netconf-server
+$(SONIC_NETCONF_SERVER)_DEPENDS  = $(SONIC_MGMT_COMMON) $(SONIC_MGMT_COMMON_CODEGEN)
+$(SONIC_NETCONF_SERVER)_RDEPENDS = 
+SONIC_DPKG_DEBS += $(SONIC_NETCONF_SERVER)
+
+SONIC_NETCONF_SERVER_DBG = sonic-netconf-server-dbg_1.0-01_$(CONFIGURED_ARCH).deb
+$(SONIC_NETCONF_SERVER_DBG)_DEPENDS += $(SONIC_NETCONF_SERVER)
+$(SONIC_NETCONF_SERVER_DBG)_RDEPENDS += $(SONIC_NETCONF_SERVER)
+$(eval $(call add_derived_package,$(SONIC_NETCONF_SERVER),$(SONIC_NETCONF_SERVER_DBG)))
+
+endif

slave.mk

@@ -1434,6 +1434,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export include_macsec="$(INCLUDE_MACSEC)"
 	export include_dhcp_server="$(INCLUDE_DHCP_SERVER)"
 	export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"
+	export include_netconf_server="$(INCLUDE_NETCONF_SERVER)"
 	export include_iccpd="$(INCLUDE_ICCPD)"
 	export pddf_support="$(PDDF_SUPPORT)"
 	export include_pde="$(INCLUDE_PDE)"