diff --git a/.gitmodules b/.gitmodules index 3d2a8e3f7a1d..7419536b2650 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,7 +34,7 @@ url = https://github.com/p4lang/ptf.git [submodule "src/sonic-utilities"] path = src/sonic-utilities - url = https://github.com/Azure/sonic-utilities + url = https://github.com/project-arlo/sonic-utilities.git [submodule "platform/broadcom/sonic-platform-modules-arista"] path = platform/broadcom/sonic-platform-modules-arista url = https://github.com/aristanetworks/sonic @@ -66,3 +66,9 @@ [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"] path = platform/mellanox/mlnx-sai/SAI-Implementation url = https://github.com/Mellanox/SAI-Implementation +[submodule "src/sonic-mgmt-framework"] + path = src/sonic-mgmt-framework + url = https://github.com/project-arlo/sonic-mgmt-framework.git +[submodule "src/telemetry"] + path = src/telemetry + url = https://github.com/project-arlo/sonic-telemetry.git diff --git a/Makefile.work b/Makefile.work index cd867a46ef53..0b14018d19d2 100644 --- a/Makefile.work +++ b/Makefile.work @@ -90,6 +90,7 @@ endif DOCKER_RUN := docker run --rm=true --privileged \ -v $(DOCKER_BUILDER_MOUNT) \ -w $(DOCKER_BUILDER_WORKDIR) \ + --tmpfs /fsroot:exec,dev \ -e "http_proxy=$(http_proxy)" \ -e "https_proxy=$(https_proxy)" \ -i$(if $(TERM),t,) diff --git a/build_debian.sh b/build_debian.sh index 92938c40b7f4..8d56929d471d 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -33,7 +33,7 @@ DOCKER_VERSION=5:18.09.2~3-0~debian-stretch LINUX_KERNEL_VERSION=4.9.0-9-2 ## Working directory to prepare the file system -FILESYSTEM_ROOT=./fsroot +FILESYSTEM_ROOT=/fsroot PLATFORM_DIR=platform ## Hostname for the linux image HOSTNAME=sonic @@ -56,9 +56,8 @@ DEFAULT_USERINFO="Default admin user,,," ## Prepare the file system directory if [[ -d $FILESYSTEM_ROOT ]]; then - sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory" + sudo chown $(id -nu):$(id -ng) $FILESYSTEM_ROOT || die "Failed to own chroot directory" fi -mkdir -p $FILESYSTEM_ROOT mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/x86_64-grub touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime @@ -426,8 +425,7 @@ fi ## Organization specific extensions such as Configuration & Scripts for features like AAA, ZTP... if [ "${enable_organization_extensions}" = "y" ]; then - if [ -f files/build_templates/organization_extensions.sh ]; then - sudo chmod 755 files/build_templates/organization_extensions.sh + if [ -x files/build_templates/organization_extensions.sh ]; then ./files/build_templates/organization_extensions.sh -f $FILESYSTEM_ROOT -h $HOSTNAME fi fi @@ -487,16 +485,19 @@ sudo umount $FILESYSTEM_ROOT/proc || true sudo mkdir $FILESYSTEM_ROOT/host ## Compress most file system into squashfs file -sudo rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS +rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS ## Output the file system total size for diag purpose ## Note: -x to skip directories on different file systems, such as /proc sudo du -hsx $FILESYSTEM_ROOT sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker -sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +sudo mksquashfs $FILESYSTEM_ROOT /tmp/$FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +cp /tmp/$FILESYSTEM_SQUASHFS $FILESYSTEM_SQUASHFS ## Compress docker files -pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C var/lib/docker .; popd +pushd $FILESYSTEM_ROOT && sudo tar czf /tmp/$FILESYSTEM_DOCKERFS -C var/lib/docker .; popd +cp /tmp/$FILESYSTEM_DOCKERFS $FILESYSTEM_DOCKERFS ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo zip $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd -sudo zip -g $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS +pushd $FILESYSTEM_ROOT && sudo zip /tmp/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd +sudo zip -g /tmp/$ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS +cp /tmp/$ONIE_INSTALLER_PAYLOAD $ONIE_INSTALLER_PAYLOAD diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 new file mode 100644 index 000000000000..ec8d1781ae3f --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -0,0 +1,44 @@ +FROM docker-config-engine-stretch + +ARG docker_container_name +RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update +RUN pip install connexion==1.1.15 \ + python_dateutil==2.6.0\ + setuptools==21.0.0 \ + grpcio-tools==1.20.0 \ + pyangbind==0.6.0 \ + certifi==2017.4.17 \ + python-dateutil==2.6.0 \ + six==1.11.0 \ + urllib3==1.21.1 + + +#RUN apt-get install -f -y libdbus-1-3 libdaemon0 libjansson4 + +## Install redis-tools dependencies +## TODO: implicitly install dependencies +RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson + +COPY \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor -%} +debs/ + +RUN dpkg -i \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor %} + +COPY ["start.sh", "rest-server.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] + +RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN rm -rf /debs + +ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh new file mode 100755 index 000000000000..f2a29c9b1ed2 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Startup script for SONiC Management REST Server + +SERVER_PORT= +LOG_LEVEL= +CLIENT_AUTH= +SERVER_CRT= +SERVER_KEY= +CA_CERT= + +# Read basic server settings from REST_SERVER|default entry +HAS_REST_CONFIG=$(sonic-cfggen -d -v "1 if REST_SERVER and REST_SERVER['default']") +if [ "$HAS_REST_CONFIG" == "1" ]; then + SERVER_PORT=$(sonic-cfggen -d -v "REST_SERVER['default']['port']") + CLIENT_AUTH=$(sonic-cfggen -d -v "REST_SERVER['default']['client_auth']") + LOG_LEVEL=$(sonic-cfggen -d -v "REST_SERVER['default']['log_level']") +fi + +# Read certificate file paths from DEVICE_METADATA|x509 entry. +HAS_X509_CONFIG=$(sonic-cfggen -d -v "1 if DEVICE_METADATA and DEVICE_METADATA['x509']") +if [ "$HAS_X509_CONFIG" == "1" ]; then + SERVER_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']") + SERVER_KEY=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_key']") + CA_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['ca_crt']") +fi + +# Create temporary server certificate if they not configured in ConfigDB +if [ -z $SERVER_CRT ] && [ -z $SERVER_KEY ]; then + echo "Generating temporary TLS server certificate ..." + (cd /tmp && /usr/sbin/generate_cert --host="localhost,127.0.0.1") + SERVER_CRT=/tmp/cert.pem + SERVER_KEY=/tmp/key.pem +fi + + +REST_SERVER_ARGS="-ui /rest_ui -logtostderr" +[ ! -z $SERVER_PORT ] && REST_SERVER_ARGS+=" -port $SERVER_PORT" +[ ! -z $LOG_LEVEL ] && REST_SERVER_ARGS+=" -v $LOG_LEVEL" +[ ! -z $CLIENT_AUTH ] && REST_SERVER_ARGS+=" -client_auth $CLIENT_AUTH" +[ ! -z $SERVER_CRT ] && REST_SERVER_ARGS+=" -cert $SERVER_CRT" +[ ! -z $SERVER_KEY ] && REST_SERVER_ARGS+=" -key $SERVER_KEY" +[ ! -z $CA_CRT ] && REST_SERVER_ARGS+=" -cacert $CA_CRT" + +echo "REST_SERVER_ARGS = $REST_SERVER_ARGS" + + +export CVL_SCHEMA_PATH=/usr/sbin/schema +export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/extensions +export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/user_types + +exec /usr/sbin/rest_server ${REST_SERVER_ARGS} diff --git a/dockers/docker-sonic-mgmt-framework/start.sh b/dockers/docker-sonic-mgmt-framework/start.sh new file mode 100755 index 000000000000..24d355670e87 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/start.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p /var/sonic +echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisorctl start rest-server diff --git a/dockers/docker-sonic-mgmt-framework/supervisord.conf b/dockers/docker-sonic-mgmt-framework/supervisord.conf new file mode 100644 index 000000000000..e26f815f5fb9 --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/supervisord.conf @@ -0,0 +1,28 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:start.sh] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rest-server] +command=/usr/bin/rest-server.sh +priority=3 +autostart=false +autorestart=true +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/files/build_templates/mgmt-framework.service.j2 b/files/build_templates/mgmt-framework.service.j2 new file mode 100644 index 000000000000..d0a030347b51 --- /dev/null +++ b/files/build_templates/mgmt-framework.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Management Framework container +Requires=swss.service +After=swss.service +Before=ntp-config.service + +[Service] +User={{ sonicadmin_user }} +ExecStartPre=/usr/bin/{{docker_container_name}}.sh start +ExecStart=/usr/bin/{{docker_container_name}}.sh wait +ExecStop=/usr/bin/{{docker_container_name}}.sh stop + +[Install] +WantedBy=multi-user.target diff --git a/files/image_config/platform/rc.local b/files/image_config/platform/rc.local index 76fe390be3cd..6e1a5d85b0d6 100755 --- a/files/image_config/platform/rc.local +++ b/files/image_config/platform/rc.local @@ -322,4 +322,13 @@ if [ -f $FIRST_BOOT_FILE ]; then firsttime_exit fi +# Create dir where following scripts put their output files +mkdir -p /var/platform + +# Write contents of system EEPROM as JSON file +/usr/bin/decode-syseeprom | /usr/bin/syseeprom-to-json > /var/platform/syseeprom + +# Start system information poller +/usr/bin/syspoll & + exit 0 diff --git a/rules/docker-sonic-mgmt-framework.mk b/rules/docker-sonic-mgmt-framework.mk new file mode 100644 index 000000000000..8e973da313b4 --- /dev/null +++ b/rules/docker-sonic-mgmt-framework.mk @@ -0,0 +1,22 @@ +# docker image for mgmt-framework + +DOCKER_MGMT-FRAMEWORK_STEM = docker-sonic-mgmt-framework +DOCKER_MGMT-FRAMEWORK = $(DOCKER_MGMT-FRAMEWORK_STEM).gz +DOCKER_MGMT-FRAMEWORK_DBG = $(DOCKER_MGMT-FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz + +$(DOCKER_MGMT-FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT-FRAMEWORK_STEM) + +$(DOCKER_MGMT-FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT-FRAMEWORK) +#$(DOCKER_MGMT-FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) + +SONIC_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) +$(DOCKER_MGMT-FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) +#$(DOCKER_MGMT-FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) + +SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT-FRAMEWORK) +SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT-FRAMEWORK) + +$(DOCKER_MGMT-FRAMEWORK)_CONTAINER_NAME = mgmt-framework +$(DOCKER_MGMT-FRAMEWORK)_RUN_OPT += --net=host --privileged -t +$(DOCKER_MGMT-FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro + diff --git a/rules/sonic-mgmt-framework.mk b/rules/sonic-mgmt-framework.mk new file mode 100644 index 000000000000..42da60c6c716 --- /dev/null +++ b/rules/sonic-mgmt-framework.mk @@ -0,0 +1,7 @@ +# SONiC mgmt-framework package + +SONIC_MGMT-FRAMEWORK = sonic-mgmt-framework_1.0-01_amd64.deb +$(SONIC_MGMT-FRAMEWORK)_SRC_PATH = $(SRC_PATH)/sonic-mgmt-framework +$(SONIC_MGMT-FRAMEWORK)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_MGMT-FRAMEWORK)_RDEPENDS = $(LIBYANG) +SONIC_DPKG_DEBS += $(SONIC_MGMT-FRAMEWORK) diff --git a/rules/telemetry.mk b/rules/telemetry.mk index a160bf50899c..a5179b438ce0 100644 --- a/rules/telemetry.mk +++ b/rules/telemetry.mk @@ -2,4 +2,6 @@ SONIC_TELEMETRY = sonic-telemetry_0.1_amd64.deb $(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/telemetry +$(SONIC_TELEMETRY)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_TELEMETRY)_RDEPENDS = $(LIBYANG) SONIC_DPKG_DEBS += $(SONIC_TELEMETRY) diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework new file mode 160000 index 000000000000..0f1e1d9d1cee --- /dev/null +++ b/src/sonic-mgmt-framework @@ -0,0 +1 @@ +Subproject commit 0f1e1d9d1cee78d76074c8fa2430dad310c8cb4d diff --git a/src/sonic-utilities b/src/sonic-utilities index 09806b861486..bd16c4467e3c 160000 --- a/src/sonic-utilities +++ b/src/sonic-utilities @@ -1 +1 @@ -Subproject commit 09806b861486091d9db5cb75bdd2cc9428e46844 +Subproject commit bd16c4467e3c4aefecce6f545529aea7cc633d6a diff --git a/src/telemetry b/src/telemetry new file mode 160000 index 000000000000..dd18f0dd8c12 --- /dev/null +++ b/src/telemetry @@ -0,0 +1 @@ +Subproject commit dd18f0dd8c1251d7d0bb054a8db32f7d54cd9b9b diff --git a/src/telemetry/Makefile b/src/telemetry/Makefile deleted file mode 100644 index 91a822d4f93c..000000000000 --- a/src/telemetry/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -export GOPATH=/tmp/go - -INSTALL := /usr/bin/install - -all: sonic-telemetry - -sonic-telemetry: - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/telemetry - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/dialout/dialout_client_cli - -install: - $(INSTALL) -D ${GOPATH}/bin/telemetry $(DESTDIR)/usr/sbin/telemetry - $(INSTALL) -D ${GOPATH}/bin/dialout_client_cli $(DESTDIR)/usr/sbin/dialout_client_cli - -deinstall: - rm $(DESTDIR)/usr/sbin/telemetry - rm $(DESTDIR)/usr/sbin/dialout_client_cli - -clean: - rm -fr ${GOPATH} diff --git a/src/telemetry/debian/changelog b/src/telemetry/debian/changelog deleted file mode 100644 index 77dab629040e..000000000000 --- a/src/telemetry/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -sonic-telemetry (0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- Jipan Yang Sat, 24 Mar 2018 12:48:22 -0700 diff --git a/src/telemetry/debian/compat b/src/telemetry/debian/compat deleted file mode 100644 index ec635144f600..000000000000 --- a/src/telemetry/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/src/telemetry/debian/control b/src/telemetry/debian/control deleted file mode 100644 index f1b6a7c07282..000000000000 --- a/src/telemetry/debian/control +++ /dev/null @@ -1,17 +0,0 @@ -Source: sonic-telemetry -Section: devel -Priority: optional -Maintainer: Jipan Yang -Build-Depends: debhelper (>= 8.0.0), - dh-systemd -Standards-Version: 3.9.3 -Homepage: https://github.com/Azure/sonic-telemetry -XS-Go-Import-Path: github.com/Azure/sonic-telemetry - -Package: sonic-telemetry -Architecture: any -Built-Using: ${misc:Built-Using} -Depends: ${misc:Depends}, - ${shlibs:Depends} -Description: SONiC telemetry - sonic-telemetry \ No newline at end of file diff --git a/src/telemetry/debian/rules b/src/telemetry/debian/rules deleted file mode 100755 index 3995a26d7fcd..000000000000 --- a/src/telemetry/debian/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/make -f -%: - dh $@ --with systemd diff --git a/src/telemetry/debian/telemetry.init.d b/src/telemetry/debian/telemetry.init.d deleted file mode 100644 index 2fea32e17c49..000000000000 --- a/src/telemetry/debian/telemetry.init.d +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: sonic-telemetry -# Required-Start: $local_fs $network $remote_fs $syslog -# Required-Stop: $local_fs $network $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop the telemetry -# Description: sonic-telemetry is an implementation of sonic telemetry daemon in Go -### END INIT INFO -# - -exit 0