From d89854f0782e2a93573672d21314e3bdda835fd6 Mon Sep 17 00:00:00 2001 From: David Pilnik Date: Mon, 11 Dec 2023 17:01:57 +0200 Subject: [PATCH 1/2] Redis acl: add hiredis makefile instead package manager installation --- rules/dhcpmon.mk | 2 +- rules/dhcprelay.mk | 2 +- rules/docker-config-engine-bullseye.mk | 3 ++- rules/docker-config-engine-buster.mk | 3 ++- rules/docker-mux.mk | 4 ++-- rules/docker-restapi.mk | 2 +- rules/docker-sonic-sdk-buildenv.mk | 1 + rules/hiredis.dep | 10 +++++++++ rules/hiredis.mk | 18 +++++++++++++++++ rules/linkmgrd.mk | 8 ++++---- rules/redis.mk | 2 ++ rules/restapi.mk | 4 ++-- rules/sflow.mk | 1 + rules/swss-common.mk | 4 ++-- sonic-slave-bullseye/Dockerfile.j2 | 2 -- sonic-slave-buster/Dockerfile.j2 | 5 ----- src/hiredis/.gitignore | 4 ++++ src/hiredis/Makefile | 28 ++++++++++++++++++++++++++ 18 files changed, 81 insertions(+), 22 deletions(-) create mode 100644 rules/hiredis.dep create mode 100644 rules/hiredis.mk create mode 100644 src/hiredis/.gitignore create mode 100644 src/hiredis/Makefile diff --git a/rules/dhcpmon.mk b/rules/dhcpmon.mk index 560061380340..8f9d6403a677 100644 --- a/rules/dhcpmon.mk +++ b/rules/dhcpmon.mk @@ -4,7 +4,7 @@ SONIC_DHCPMON_VERSION = 1.0.0-0 SONIC_DHCPMON_PKG_NAME = dhcpmon SONIC_DHCPMON = sonic-$(SONIC_DHCPMON_PKG_NAME)_$(SONIC_DHCPMON_VERSION)_$(CONFIGURED_ARCH).deb -$(SONIC_DHCPMON)_DEPENDS = $(LIBSWSSCOMMON) $(LIBSWSSCOMMON_DEV) +$(SONIC_DHCPMON)_DEPENDS = $(LIBSWSSCOMMON) $(LIBHIREDIS) $(LIBSWSSCOMMON_DEV) $(LIBHIREDIS_DEV) $(SONIC_DHCPMON)_SRC_PATH = $(SRC_PATH)/$(SONIC_DHCPMON_PKG_NAME) SONIC_DPKG_DEBS += $(SONIC_DHCPMON) diff --git a/rules/dhcprelay.mk b/rules/dhcprelay.mk index 80c3cb6341f3..0a32d7d54297 100644 --- a/rules/dhcprelay.mk +++ b/rules/dhcprelay.mk @@ -4,7 +4,7 @@ SONIC_DHCPRELAY_VERSION = 1.0.0-0 SONIC_DHCPRELAY_PKG_NAME = dhcp6relay SONIC_DHCPRELAY = sonic-$(SONIC_DHCPRELAY_PKG_NAME)_$(SONIC_DHCPRELAY_VERSION)_$(CONFIGURED_ARCH).deb -$(SONIC_DHCPRELAY)_DEPENDS = $(LIBSWSSCOMMON) $(LIBSWSSCOMMON_DEV) +$(SONIC_DHCPRELAY)_DEPENDS = $(LIBSWSSCOMMON) $(LIBHIREDIS) $(LIBSWSSCOMMON_DEV) $(LIBHIREDIS_DEV) $(SONIC_DHCPRELAY)_SRC_PATH = $(SRC_PATH)/dhcprelay SONIC_DPKG_DEBS += $(SONIC_DHCPRELAY) diff --git a/rules/docker-config-engine-bullseye.mk b/rules/docker-config-engine-bullseye.mk index aa91a56279ce..5a92849a120b 100644 --- a/rules/docker-config-engine-bullseye.mk +++ b/rules/docker-config-engine-bullseye.mk @@ -23,7 +23,8 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS = $($(DOCKER_BASE_BULLSEYE)_DBG_DEPENDS) \ - $(LIBSWSSCOMMON_DBG) + $(LIBSWSSCOMMON_DBG) \ + $(LIBHIREDIS_DBG) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BULLSEYE)_DBG_IMAGE_PACKAGES) SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE_BULLSEYE) diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index 9f4035d6aa24..0f787de2abc3 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -22,7 +22,8 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) \ - $(LIBSWSSCOMMON_DBG) + $(LIBSWSSCOMMON_DBG) \ + $(LIBHIREDIS_DBG) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BUSTER)_DBG_IMAGE_PACKAGES) SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE_BUSTER) diff --git a/rules/docker-mux.mk b/rules/docker-mux.mk index fae8d217888c..251877d99241 100644 --- a/rules/docker-mux.mk +++ b/rules/docker-mux.mk @@ -6,9 +6,9 @@ DOCKER_MUX_DBG = $(DOCKER_MUX_STEM)-$(DBG_IMAGE_MARK).gz $(DOCKER_MUX)_PATH = $(DOCKERS_PATH)/$(DOCKER_MUX_STEM) -$(DOCKER_MUX)_DEPENDS = $(SONIC_LINKMGRD) $(LIBSWSSCOMMON) +$(DOCKER_MUX)_DEPENDS = $(SONIC_LINKMGRD) $(LIBSWSSCOMMON) $(LIBHIREDIS) $(DOCKER_MUX)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS) -$(DOCKER_MUX)_DBG_DEPENDS += $(SONIC_LINKMGRD_DBG) $(LIBSWSSCOMMON_DBG) +$(DOCKER_MUX)_DBG_DEPENDS += $(SONIC_LINKMGRD_DBG) $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DBG) $(DOCKER_MUX)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_IMAGE_PACKAGES) diff --git a/rules/docker-restapi.mk b/rules/docker-restapi.mk index e6dc2dcf47ea..be22e1b386d3 100644 --- a/rules/docker-restapi.mk +++ b/rules/docker-restapi.mk @@ -3,7 +3,7 @@ DOCKER_RESTAPI_STEM = docker-sonic-restapi DOCKER_RESTAPI = $(DOCKER_RESTAPI_STEM).gz -$(DOCKER_RESTAPI)_DEPENDS += $(LIBNL3) $(LIBNL_GENL3) \ +$(DOCKER_RESTAPI)_DEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \ $(LIBNL_ROUTE3) $(LIBSWSSCOMMON) $(RESTAPI) $(DOCKER_RESTAPI)_PATH = $(DOCKERS_PATH)/$(DOCKER_RESTAPI_STEM) diff --git a/rules/docker-sonic-sdk-buildenv.mk b/rules/docker-sonic-sdk-buildenv.mk index 2ab1a99d9b72..17b71a8dea36 100644 --- a/rules/docker-sonic-sdk-buildenv.mk +++ b/rules/docker-sonic-sdk-buildenv.mk @@ -11,6 +11,7 @@ $(DOCKER_SONIC_SDK_BUILDENV)_DEPENDS += $(LIBSAIVS) \ $(LIBSAIREDIS_DEV) \ $(LIBSAIMETADATA_DEV) \ $(LIBSWSSCOMMON_DEV) \ + $(LIBHIREDIS_DEV) \ $(LIBNL3_DEV) \ $(LIBNL_GENL3_DEV) \ $(LIBNL_ROUTE3_DEV) \ diff --git a/rules/hiredis.dep b/rules/hiredis.dep new file mode 100644 index 000000000000..93bea540289f --- /dev/null +++ b/rules/hiredis.dep @@ -0,0 +1,10 @@ + +SPATH := $($(LIBHIREDIS)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/hiredis.mk rules/hiredis.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(LIBHIREDIS)_CACHE_MODE := GIT_CONTENT_SHA +$(LIBHIREDIS)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(LIBHIREDIS)_DEP_FILES := $(DEP_FILES) + diff --git a/rules/hiredis.mk b/rules/hiredis.mk new file mode 100644 index 000000000000..c1e0b2fe76f1 --- /dev/null +++ b/rules/hiredis.mk @@ -0,0 +1,18 @@ +# libhiredis package + +HIREDIS_VERSION = 1.2.0 +HIREDIS_VERSION_FULL = ${HIREDIS_VERSION}-4 + +export HIREDIS_VERSION HIREDIS_VERSION_FULL + +LIBHIREDIS = libhiredis1.1.0_$(HIREDIS_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(LIBHIREDIS)_SRC_PATH = $(SRC_PATH)/hiredis +SONIC_MAKE_DEBS += $(LIBHIREDIS) + +LIBHIREDIS_DEV = libhiredis-dev_$(HIREDIS_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(eval $(call add_derived_package,$(LIBHIREDIS),$(LIBHIREDIS_DEV))) + +LIBHIREDIS_DBG = libhiredis1.1.0-dbgsym_$(HIREDIS_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(eval $(call add_derived_package,$(LIBHIREDIS),$(LIBHIREDIS_DBG))) + +export LIBHIREDIS LIBHIREDIS_DEV LIBHIREDIS_DBG \ No newline at end of file diff --git a/rules/linkmgrd.mk b/rules/linkmgrd.mk index 6a53b579edd9..07101f0c47fb 100644 --- a/rules/linkmgrd.mk +++ b/rules/linkmgrd.mk @@ -7,14 +7,14 @@ export SONIC_LINKMGRD_VERSION SONIC_LINKMGRD_PKG_NAME SONIC_LINKMGRD = sonic-$(SONIC_LINKMGRD_PKG_NAME)_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb $(SONIC_LINKMGRD)_SRC_PATH = $(SRC_PATH)/$(SONIC_LINKMGRD_PKG_NAME) -$(SONIC_LINKMGRD)_DEPENDS = $(LIBSWSSCOMMON_DEV) -$(SONIC_LINKMGRD)_RDEPENDS = $(LIBSWSSCOMMON) +$(SONIC_LINKMGRD)_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBHIREDIS_DEV) +$(SONIC_LINKMGRD)_RDEPENDS = $(LIBSWSSCOMMON) $(LIBHIREDIS) SONIC_DPKG_DEBS += $(SONIC_LINKMGRD) SONIC_LINKMGRD_DBG = sonic-$(SONIC_LINKMGRD_PKG_NAME)-dbgsym_$(SONIC_LINKMGRD_VERSION)_$(CONFIGURED_ARCH).deb -$(SONIC_LINKMGRD)_DBG_DEPENDS = $(LIBSWSSCOMMON_DEV) -$(SONIC_LINKMGRD)_DBG_RDEPENDS = $(LIBSWSSCOMMON_DBG) +$(SONIC_LINKMGRD)_DBG_DEPENDS = $(LIBSWSSCOMMON_DEV) $(LIBHIREDIS_DEV) +$(SONIC_LINKMGRD)_DBG_RDEPENDS = $(LIBSWSSCOMMON_DBG) $(LIBHIREDIS_DBG) $(eval $(call add_derived_package,$(SONIC_LINKMGRD),$(SONIC_LINKMGRD_DBG))) export SONIC_LINKMGRD SONIC_LINKMGRD_DBG diff --git a/rules/redis.mk b/rules/redis.mk index f7f9ab41a307..53566b5cdf8b 100644 --- a/rules/redis.mk +++ b/rules/redis.mk @@ -7,6 +7,8 @@ ifneq ($(BLDENV),buster) REDIS_TOOLS = redis-tools_$(REDIS_VERSION)_$(CONFIGURED_ARCH).deb $(REDIS_TOOLS)_SRC_PATH = $(SRC_PATH)/redis + $(REDIS_TOOLS)_DEPENDS += $(LIBHIREDIS_DEV) + $(REDIS_TOOLS)_RDEPENDS += $(LIBHIREDIS) SONIC_MAKE_DEBS += $(REDIS_TOOLS) REDIS_TOOLS_DBG = redis-tools-dbgsym_$(REDIS_VERSION)_$(CONFIGURED_ARCH).deb diff --git a/rules/restapi.mk b/rules/restapi.mk index 9be5be9ae06a..21eed4cfbb51 100644 --- a/rules/restapi.mk +++ b/rules/restapi.mk @@ -2,8 +2,8 @@ RESTAPI = sonic-rest-api_1.0.1_$(CONFIGURED_ARCH).deb $(RESTAPI)_SRC_PATH = $(SRC_PATH)/sonic-restapi -$(RESTAPI)_DEPENDS += $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \ +$(RESTAPI)_DEPENDS += $(LIBHIREDIS_DEV) $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \ $(LIBNL_ROUTE3_DEV) $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON) -$(RESTAPI)_RDEPENDS += $(LIBNL3) $(LIBNL_GENL3) \ +$(RESTAPI)_RDEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \ $(LIBNL_ROUTE3) $(LIBSWSSCOMMON) SONIC_DPKG_DEBS += $(RESTAPI) diff --git a/rules/sflow.mk b/rules/sflow.mk index a7dc205aca2c..53dac559fcd6 100644 --- a/rules/sflow.mk +++ b/rules/sflow.mk @@ -7,6 +7,7 @@ export ENABLE_SFLOW_DROPMON export HSFLOWD_VERSION HSFLOWD_SUBVERSION HSFLOWD = hsflowd_$(HSFLOWD_VERSION)-$(HSFLOWD_SUBVERSION)_$(CONFIGURED_ARCH).deb +$(HSFLOWD)_DEPENDS += $(LIBHIREDIS_DEV) $(HSFLOWD)_SRC_PATH = $(SRC_PATH)/sflow/hsflowd SONIC_MAKE_DEBS += $(HSFLOWD) diff --git a/rules/swss-common.mk b/rules/swss-common.mk index 5d657d5e500b..838c9a6725da 100644 --- a/rules/swss-common.mk +++ b/rules/swss-common.mk @@ -7,10 +7,10 @@ LIBSWSSCOMMON = $(LIBSWSSCOMMON_NAME)_$(LIBSWSSCOMMON_VERSION)_$(CONFIGURED_ARCH $(LIBSWSSCOMMON)_SRC_PATH = $(SRC_PATH)/sonic-swss-common $(LIBSWSSCOMMON)_VERSION = $(LIBSWSSCOMMON_VERSION) $(LIBSWSSCOMMON)_NAME = $(LIBSWSSCOMMON_NAME) -$(LIBSWSSCOMMON)_DEPENDS += $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \ +$(LIBSWSSCOMMON)_DEPENDS += $(LIBHIREDIS_DEV) $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \ $(LIBNL_ROUTE3_DEV) $(LIBNL_NF3_DEV) \ $(LIBNL_CLI_DEV) $(LIBYANG_DEV) $(LIBYANG) -$(LIBSWSSCOMMON)_RDEPENDS += $(LIBNL3) $(LIBNL_GENL3) \ +$(LIBSWSSCOMMON)_RDEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \ $(LIBNL_ROUTE3) $(LIBNL_NF3) $(LIBNL_CLI) $(LIBYANG) SONIC_DPKG_DEBS += $(LIBSWSSCOMMON) diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index 7bcf438cca82..6021eeaa6f1a 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -414,8 +414,6 @@ RUN apt-get update && apt-get install -y eatmydata && eatmydata apt-get install libgirepository1.0-dev \ libsystemd-dev \ pkg-config \ -# For sonic-swss-common build - libhiredis-dev \ # For audisp-tacplus libauparse-dev \ auditd \ diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 2d582f659fb0..f4a690d2782c 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -368,9 +368,6 @@ RUN apt-get update && apt-get install -y eatmydata && eatmydata apt-get install # For WPA supplication qtbase5-dev \ aspell-en \ -{%- if CROSS_BUILD_ENVIRON != "y" %} - libhiredis-dev \ -{%- endif %} swig3.0 \ libpython2.7-dev \ libssl-dev \ @@ -413,8 +410,6 @@ RUN apt-get update && apt-get install -y eatmydata && eatmydata apt-get install libgirepository1.0-dev \ libsystemd-dev \ pkg-config \ -# For sonic-swss-common build - libhiredis-dev \ # For audisp-tacplus libauparse-dev \ auditd diff --git a/src/hiredis/.gitignore b/src/hiredis/.gitignore new file mode 100644 index 000000000000..6817ac576f28 --- /dev/null +++ b/src/hiredis/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!Makefile + diff --git a/src/hiredis/Makefile b/src/hiredis/Makefile new file mode 100644 index 000000000000..1b99b465f739 --- /dev/null +++ b/src/hiredis/Makefile @@ -0,0 +1,28 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e +export USE_SSL=1 + +MAIN_TARGET = $(LIBHIREDIS) +DERIVED_TARGETS = $(LIBHIREDIS_DBG) $(LIBHIREDIS_DEV) + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + # Remove any stale files + rm -rf ./hiredis-$(HIREDIS_VERSION) + + # Get hiredis release, debian files + dget -u http://http.debian.net/debian/pool/main/h/hiredis/hiredis_$(HIREDIS_VERSION_FULL).dsc + + # Build source and Debian packages + pushd ./hiredis-$(HIREDIS_VERSION) +ifeq ($(CROSS_BUILD_ENVIRON), y) + DEB_BUILD_OPTIONS="USE_SSL=1" dpkg-buildpackage -rfakeroot -d -b -us -uc -a$(CONFIGURED_ARCH) -Pcross,nocheck -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) +else + DEB_BUILD_OPTIONS="USE_SSL=1" dpkg-buildpackage -rfakeroot -d -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) +endif + popd + + # Move the newly-built .deb packages to the destination directory + mv $* $(DERIVED_TARGETS) $(DEST)/ + +$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) From a8856f9b31c7f57c9c2df729282af7226a95f496 Mon Sep 17 00:00:00 2001 From: David Pilnik Date: Thu, 7 Dec 2023 12:59:55 +0200 Subject: [PATCH 2/2] Add Redis ACL support --- dockers/docker-database/Dockerfile.j2 | 12 ++++- .../docker-database/docker-database-init.sh | 10 ++++ dockers/docker-database/supervisord.conf.j2 | 3 +- dockers/docker-database/users.acl.template | 3 ++ dockers/docker-orchagent/buffermgrd.sh | 2 +- files/build_templates/docker_image_ctl.j2 | 2 + .../build_templates/sonic_debian_extension.j2 | 13 +++++ files/image_config/pcie-check/pcie-check.sh | 4 +- files/image_config/platform/rc.local | 31 +++++++++++ files/scripts/gen-redis-certs.sh | 52 +++++++++++++++++++ .../sonic_platform/module.py | 8 ++- platform/vs/docker-sonic-vs/buffermgrd.sh | 2 +- platform/vs/tests/bgp/test_default_route.py | 4 +- src/sonic-config-engine/tests/test_cfggen.py | 9 ++++ .../tests/test_cfggen_platformJson.py | 1 + src/sonic-config-engine/tests/test_j2files.py | 6 +++ .../tests/test_minigraph_case.py | 7 +++ .../tests/test_multinpu_cfggen.py | 17 +++--- 18 files changed, 169 insertions(+), 17 deletions(-) create mode 100644 dockers/docker-database/users.acl.template create mode 100644 files/scripts/gen-redis-certs.sh diff --git a/dockers/docker-database/Dockerfile.j2 b/dockers/docker-database/Dockerfile.j2 index 3b0de019ef68..5ecb59d9aed4 100644 --- a/dockers/docker-database/Dockerfile.j2 +++ b/dockers/docker-database/Dockerfile.j2 @@ -23,6 +23,7 @@ RUN pip3 install click {{ install_debian_packages(docker_database_debs.split(' ')) }} {%- endif %} +ENV REDIS_SHADOW_TLS=/etc/ssl/certs_redis/certs/tls/ # Clean up RUN apt-get clean -y && \ apt-get autoclean -y && \ @@ -30,11 +31,19 @@ RUN apt-get clean -y && \ rm -rf /debs ~/.cache && \ sed -ri 's/^(save .*$)/# \1/g; \ s/^daemonize yes$/daemonize no/; \ - s/^logfile .*$/logfile ""/; \ + s|^logfile .*$|logfile /etc/redis/redis-server.log|; \ s/^# syslog-enabled no$/syslog-enabled no/; \ s/^# unixsocket/unixsocket/; \ s/redis-server.sock/redis.sock/g; \ s/^client-output-buffer-limit pubsub [0-9]+mb [0-9]+mb [0-9]+/client-output-buffer-limit pubsub 0 0 0/; \ + s/^port 6379/# port 6379/; \ + s/^# port 0/port 0/; \ + s/^# tls-port 6379/tls-port 6379/; \ + /tls-auth-clients no/s/^# //; \ + s|# tls-cert-file .*|tls-cert-file '"$REDIS_SHADOW_TLS"'/redis.crt|; \ + s|# tls-key-file .*|tls-key-file '"$REDIS_SHADOW_TLS"'/redis.key|; \ + s|# tls-ca-cert-file .*|tls-ca-cert-file '"$REDIS_SHADOW_TLS"'/ca.crt|; \ + /aclfile \/etc\/redis\/users.acl/s/^# //; \ s/^notify-keyspace-events ""$/notify-keyspace-events AKE/; \ s/^databases [0-9]+$/databases 100/ \ ' /etc/redis/redis.conf @@ -48,5 +57,6 @@ COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] COPY ["files/sysctl-net.conf", "/etc/sysctl.d/"] COPY ["files/update_chassisdb_config", "/usr/local/bin/"] COPY ["flush_unused_database", "/usr/local/bin/"] +COPY ["users.acl.template", "/etc/redis/"] ENTRYPOINT ["/usr/local/bin/docker-database-init.sh"] diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index f6f8522bb944..db7b96131451 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -115,4 +115,14 @@ ln -sf /usr/share/zoneinfo/$TZ /etc/localtime chown -R redis:redis $REDIS_DIR +# Redis PW update in users.acl +acl_template=$(< /etc/redis/users.acl.template) + +USER_COUNTER_PASSWORD=$(cat /etc/shadow_redis_dir/shadow_redis_admin) +acl_new_admin_user="${acl_template//\$\{USER_COUNTER_PASSWORD\}/$USER_COUNTER_PASSWORD}" + +MONITOR_PASSWORD=$(cat /etc/shadow_redis_dir/shadow_redis_monitor) +acl_new_admin_monitor_users="${acl_new_admin_user//\$\{MONITOR_PASSWORD\}/$MONITOR_PASSWORD}" +echo "$acl_new_admin_monitor_users" > /etc/redis/users.acl + exec /usr/local/bin/supervisord diff --git a/dockers/docker-database/supervisord.conf.j2 b/dockers/docker-database/supervisord.conf.j2 index b71a4c59dbef..d71171fdcf84 100644 --- a/dockers/docker-database/supervisord.conf.j2 +++ b/dockers/docker-database/supervisord.conf.j2 @@ -36,9 +36,8 @@ dependent_startup=true {%- else -%} {%- set LOOPBACK_IP = '' -%} {%- endif -%} -command=/bin/bash -c "{ [[ -s /var/lib/{{ redis_inst }}/dump.rdb ]] || rm -f /var/lib/{{ redis_inst }}/dump.rdb; } && mkdir -p /var/lib/{{ redis_inst }} && exec /usr/bin/redis-server /etc/redis/redis.conf --bind {{ LOOPBACK_IP }} {{ redis_items['hostname'] }} --port {{ redis_items['port'] }} --unixsocket {{ redis_items['unix_socket_path'] }} --pidfile /var/run/redis/{{ redis_inst }}.pid --dir /var/lib/{{ redis_inst }}" +command=/bin/bash -c "{ [[ -s /var/lib/{{ redis_inst }}/dump.rdb ]] || rm -f /var/lib/{{ redis_inst }}/dump.rdb; } && mkdir -p /var/lib/{{ redis_inst }} && exec /usr/bin/redis-server /etc/redis/redis.conf --bind {{ LOOPBACK_IP }} {{ redis_items['hostname'] }} --unixsocket {{ redis_items['unix_socket_path'] }} --pidfile /var/run/redis/{{ redis_inst }}.pid --dir /var/lib/{{ redis_inst }}" priority=2 -user=redis autostart=true autorestart=false stdout_logfile=syslog diff --git a/dockers/docker-database/users.acl.template b/dockers/docker-database/users.acl.template new file mode 100644 index 000000000000..7ae0a63453cf --- /dev/null +++ b/dockers/docker-database/users.acl.template @@ -0,0 +1,3 @@ +user admin on +@all -DEBUG ~* >${USER_COUNTER_PASSWORD} +user monitor on +hgetall +keys +select +get +mget +hget -DEBUG ~* >${MONITOR_PASSWORD} +user default off \ No newline at end of file diff --git a/dockers/docker-orchagent/buffermgrd.sh b/dockers/docker-orchagent/buffermgrd.sh index b5ddaab7df0e..1fc9c3e5a84f 100755 --- a/dockers/docker-orchagent/buffermgrd.sh +++ b/dockers/docker-orchagent/buffermgrd.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUFFER_CALCULATION_MODE=$(redis-cli -n 4 hget "DEVICE_METADATA|localhost" buffer_model) +BUFFER_CALCULATION_MODE=$(sonic-db-cli CONFIG_DB hget "DEVICE_METADATA|localhost" buffer_model) if [ "$BUFFER_CALCULATION_MODE" == "dynamic" ]; then BUFFERMGRD_ARGS="-a /etc/sonic/asic_table.json" diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 95c2e1fa52a8..6da406b3da32 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -575,6 +575,7 @@ start() { # TODO: Mellanox will remove the --tmpfs exception after SDK socket path changed in new SDK version {%- endif %} docker create {{docker_image_run_opt}} \ + -v /etc/shadow_redis_dir:/etc/shadow_redis_dir:ro \ {%- if docker_container_name != "dhcp_server" %} --net=$NET \ {%- endif %} @@ -627,6 +628,7 @@ start() { {%- endif %} {%- if docker_container_name == "database" %} $DB_OPT \ + -v /etc/ssl/certs_redis:/etc/ssl/certs_redis:ro \ {%- else %} -v /var/run/redis$DEV:/var/run/redis:rw \ -v /var/run/redis-chassis:/var/run/redis-chassis:ro \ diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 377c4116fabf..8fc27fa1db01 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -113,6 +113,12 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in # Install j2cli for handling jinja template sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install j2cli +# Create an empty Redis dir for the generation of Redis ACL passwords +sudo mkdir $FILESYSTEM_ROOT/etc/shadow_redis_dir + +# Create an empty Redis dir for the public cacert of Redis TLS +sudo mkdir $FILESYSTEM_ROOT/etc/shadow_redis_dir/certs_redis + # Install Python client for Redis sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install "redis==3.5.3" @@ -932,6 +938,13 @@ sudo LANG=C cp $SCRIPTS_DIR/mgmt-framework.sh $FILESYSTEM_ROOT/usr/local/bin/mgm sudo LANG=C cp $SCRIPTS_DIR/asic_status.sh $FILESYSTEM_ROOT/usr/local/bin/asic_status.sh sudo LANG=C cp $SCRIPTS_DIR/asic_status.py $FILESYSTEM_ROOT/usr/local/bin/asic_status.py +# Copy Redis Certificate generator script +FSROOT_ETC_SSL_CERTS_REDIS=$FILESYSTEM_ROOT/etc/ssl/certs_redis +sudo LANG=C mkdir $FSROOT_ETC_SSL_CERTS_REDIS +sudo LANG=C cp $SCRIPTS_DIR/gen-redis-certs.sh $FSROOT_ETC_SSL_CERTS_REDIS/gen-redis-certs.sh +sudo chroot $FILESYSTEM_ROOT chown admin:admin /etc/ssl/certs_redis/gen-redis-certs.sh +sudo chmod 770 $FSROOT_ETC_SSL_CERTS_REDIS/gen-redis-certs.sh + # Copy sonic-netns-exec script sudo LANG=C cp $SCRIPTS_DIR/sonic-netns-exec $FILESYSTEM_ROOT/usr/bin/sonic-netns-exec diff --git a/files/image_config/pcie-check/pcie-check.sh b/files/image_config/pcie-check/pcie-check.sh index 3d4184c8684c..379409d8ec4e 100755 --- a/files/image_config/pcie-check/pcie-check.sh +++ b/files/image_config/pcie-check/pcie-check.sh @@ -38,7 +38,7 @@ function check_and_rescan_pcie_devices() fi if [ "$(eval $PCIE_CHK_CMD)" = "$EXPECTED" ]; then - redis-cli -n 6 HSET $PCIE_STATUS_TABLE "status" "PASSED" + sonic-db-cli STATE_DB HSET $PCIE_STATUS_TABLE "status" "PASSED" debug "PCIe check passed" exit else @@ -54,7 +54,7 @@ function check_and_rescan_pcie_devices() done debug "PCIe check failed" - redis-cli -n 6 HSET $PCIE_STATUS_TABLE "status" "FAILED" + sonic-db-cli STATE_DB HSET $PCIE_STATUS_TABLE "status" "FAILED" } check_and_rescan_pcie_devices diff --git a/files/image_config/platform/rc.local b/files/image_config/platform/rc.local index 5cdefa887739..234a267057a5 100755 --- a/files/image_config/platform/rc.local +++ b/files/image_config/platform/rc.local @@ -240,6 +240,37 @@ fi program_console_speed +# Generate password for Redis DB +echo "Redis PW generation" +set +x +REDIS_SHADOW_PATH=/etc/shadow_redis_dir +openssl_random_cmd=$(openssl rand -base64 32) +ADMIN_PASSWORD='' +ADMIN_PASSWORD=$(echo "$openssl_random_cmd" | tr -d '\n') +REDIS_SHADOW_ADMIN_PATH=$REDIS_SHADOW_PATH/shadow_redis_admin +touch "$REDIS_SHADOW_ADMIN_PATH" +chown admin:admin "$REDIS_SHADOW_ADMIN_PATH" +chmod 640 "$REDIS_SHADOW_ADMIN_PATH" +echo "$ADMIN_PASSWORD" > "$REDIS_SHADOW_ADMIN_PATH" +MONITOR_PASSWORD='' +MONITOR_PASSWORD=$(echo "$openssl_random_cmd" | tr -d '\n') +echo "$MONITOR_PASSWORD" > $REDIS_SHADOW_PATH/shadow_redis_monitor + +# TLS support +ETC_SSL=/etc/ssl/certs_redis +REDIS_CERTS=$ETC_SSL/certs +REDIS_CERTS_TLS=$REDIS_CERTS/tls + +# Check if the directory exists and remove it if it does +[ -d "$REDIS_CERTS" ] && rm -rf "$REDIS_CERTS" + +(cd $ETC_SSL && ./gen-redis-certs.sh) + +# Copy CA cert to host share location for Redis client usage +cp $REDIS_CERTS_TLS/ca.crt $REDIS_SHADOW_PATH/certs_redis + +set -x + if [ -f $FIRST_BOOT_FILE ]; then echo "First boot detected. Performing first boot tasks..." diff --git a/files/scripts/gen-redis-certs.sh b/files/scripts/gen-redis-certs.sh new file mode 100644 index 000000000000..5207037f661d --- /dev/null +++ b/files/scripts/gen-redis-certs.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# from redis official repo https://github.com/redis/redis/ +# Generate some test certificates which are used by the regression test suite: +# +# certs/tls/ca.{crt,key} Self signed CA certificate. +# certs/tls/redis.{crt,key} A certificate with no key usage/policy restrictions. + +generate_cert() { + local name=$1 + local cn="$2" + local opts="$3" + + local keyfile=certs/tls/${name}.key + local certfile=certs/tls/${name}.crt + + [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 + openssl req \ + -new -sha256 \ + -subj "/O=Redis Test/CN=$cn" \ + -key $keyfile | \ + openssl x509 \ + -req -sha256 \ + -CA certs/tls/ca.crt \ + -CAkey certs/tls/ca.key \ + -CAserial certs/tls/ca.txt \ + -CAcreateserial \ + -days 365 \ + $opts \ + -out $certfile +} + +mkdir -p certs/tls +[ -f certs/tls/ca.key ] || openssl genrsa -out certs/tls/ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key certs/tls/ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out certs/tls/ca.crt + +cat > certs/tls/openssl.cnf <<_END_ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server + +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client +_END_ + +generate_cert redis "Generic-cert" diff --git a/platform/mellanox/mlnx-platform-api/sonic_platform/module.py b/platform/mellanox/mlnx-platform-api/sonic_platform/module.py index 765314d7baf0..ddc4e3c582f2 100644 --- a/platform/mellanox/mlnx-platform-api/sonic_platform/module.py +++ b/platform/mellanox/mlnx-platform-api/sonic_platform/module.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. +# Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. # Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,7 +35,11 @@ class Module(ModuleBase): STATE_DB = 6 STATE_MODULAR_CHASSIS_SLOT_TABLE = 'MODULAR_CHASSIS_SLOT|{}' FIELD_SEQ_NO = 'seq_no' - redis_client = redis.Redis(db = STATE_DB) + USERNAME = 'admin' + PASSWORD = utils.read_str_from_file('/etc/shadow_redis_dir/shadow_redis_admin') + REDIS_SHADOW_TLS_CA="/etc/shadow_redis_dir/certs_redis/ca.crt" + redis_client = redis.Redis(port=6379, db=STATE_DB, username=USERNAME, password=PASSWORD, ssl=True, ssl_cert_reqs=None, ssl_ca_certs=REDIS_SHADOW_TLS_CA) + def __init__(self, slot_id): super(Module, self).__init__() diff --git a/platform/vs/docker-sonic-vs/buffermgrd.sh b/platform/vs/docker-sonic-vs/buffermgrd.sh index a36fc7f94ca8..a5c3b4fddb41 100755 --- a/platform/vs/docker-sonic-vs/buffermgrd.sh +++ b/platform/vs/docker-sonic-vs/buffermgrd.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUFFER_CALCULATION_MODE=$(redis-cli -n 4 hget "DEVICE_METADATA|localhost" buffer_model) +BUFFER_CALCULATION_MODE=$(sonic-db-cli CONFIG_DB hget "DEVICE_METADATA|localhost" buffer_model) export ASIC_VENDOR=vs if [ "$BUFFER_CALCULATION_MODE" == "dynamic" ]; then diff --git a/platform/vs/tests/bgp/test_default_route.py b/platform/vs/tests/bgp/test_default_route.py index 2cc9021ba78e..94ad412c04c2 100644 --- a/platform/vs/tests/bgp/test_default_route.py +++ b/platform/vs/tests/bgp/test_default_route.py @@ -25,7 +25,7 @@ def test_DefaultRoute(dvs, testlog): time.sleep(10) - (exit_code, output) = dvs.runcmd(["redis-cli", "hgetall", "ROUTE_TABLE:0.0.0.0/0"]) + (exit_code, output) = dvs.runcmd(["sonic-db-cli", "APPL_DB", "hgetall", "ROUTE_TABLE:0.0.0.0/0"]) print(exit_code, output) # make sure 10.10.10.1 is the correct next hop for default route @@ -34,7 +34,7 @@ def test_DefaultRoute(dvs, testlog): # insert default route for table default dvs.runcmd("ip route add default via 172.17.0.1 table default") - (exit_code, output) = dvs.runcmd(["redis-cli", "hgetall", "ROUTE_TABLE:0.0.0.0/0"]) + (exit_code, output) = dvs.runcmd(["sonic-db-cli", "APPL_DB", "hgetall", "ROUTE_TABLE:0.0.0.0/0"]) print(exit_code, output) time.sleep(10) diff --git a/src/sonic-config-engine/tests/test_cfggen.py b/src/sonic-config-engine/tests/test_cfggen.py index b8480cf81fee..bbdee9a73547 100644 --- a/src/sonic-config-engine/tests/test_cfggen.py +++ b/src/sonic-config-engine/tests/test_cfggen.py @@ -5,11 +5,18 @@ from unittest import TestCase +import sys +if sys.version_info.major == 3: + from unittest import mock +else: + import mock + TOR_ROUTER = 'ToRRouter' BACKEND_TOR_ROUTER = 'BackEndToRRouter' LEAF_ROUTER = 'LeafRouter' BACKEND_LEAF_ROUTER = 'BackEndLeafRouter' +@mock.patch('swsssdk.util.read_from_file', mock.MagicMock(return_value='mock_password')) class TestCfgGen(TestCase): def setUp(self): @@ -219,6 +226,8 @@ def test_template_json_batch_mode(self): def test_minigraph_acl(self): argument = ['-m', self.sample_graph_t0, '-p', self.port_config, '-v', 'ACL_TABLE'] output = self.run_script(argument, True, True) + # ignore safe error in unittest infra, since this file exists only in a runtime device + output = output.strip().replace("ERROR:root:Failed to read from /etc/shadow_redis_dir/shadow_redis_admin, errno is [Errno 2] No such file or directory: '/etc/shadow_redis_dir/shadow_redis_admin'", '') self.assertEqual( utils.to_dict(output.strip().replace("Warning: Ignoring Control Plane ACL NTP_ACL without type\n", '')), utils.to_dict( diff --git a/src/sonic-config-engine/tests/test_cfggen_platformJson.py b/src/sonic-config-engine/tests/test_cfggen_platformJson.py index 5d39fd2f3660..4fd367f34f69 100644 --- a/src/sonic-config-engine/tests/test_cfggen_platformJson.py +++ b/src/sonic-config-engine/tests/test_cfggen_platformJson.py @@ -17,6 +17,7 @@ # Global Variable PLATFORM_OUTPUT_FILE = "platform_output.json" +@mock.patch('swsssdk.util.read_from_file', mock.MagicMock(return_value='mock_password')) class TestCfgGenPlatformJson(TestCase): def setUp(self): diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index 6a660bec28b6..40feb3c6294d 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -7,7 +7,13 @@ import tests.common_utils as utils from sonic_py_common.general import getstatusoutput_noshell, getstatusoutput_noshell_pipe +import sys +if sys.version_info.major == 3: + from unittest import mock +else: + import mock +@mock.patch('swsssdk.util.read_from_file', mock.MagicMock(return_value='mock_password')) class TestJ2Files(TestCase): def setUp(self): self.test_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/src/sonic-config-engine/tests/test_minigraph_case.py b/src/sonic-config-engine/tests/test_minigraph_case.py index 70c5c410332e..ca6bc9cb07ef 100644 --- a/src/sonic-config-engine/tests/test_minigraph_case.py +++ b/src/sonic-config-engine/tests/test_minigraph_case.py @@ -7,10 +7,17 @@ from unittest import TestCase +import sys +if sys.version_info.major == 3: + from unittest import mock +else: + import mock + TOR_ROUTER = 'ToRRouter' BACKEND_TOR_ROUTER = 'BackEndToRRouter' BMC_MGMT_TOR_ROUTER = 'BmcMgmtToRRouter' +@mock.patch('swsssdk.util.read_from_file', mock.MagicMock(return_value='mock_password')) class TestCfgGenCaseInsensitive(TestCase): def setUp(self): diff --git a/src/sonic-config-engine/tests/test_multinpu_cfggen.py b/src/sonic-config-engine/tests/test_multinpu_cfggen.py index bc4227f85d52..1061551f4527 100644 --- a/src/sonic-config-engine/tests/test_multinpu_cfggen.py +++ b/src/sonic-config-engine/tests/test_multinpu_cfggen.py @@ -10,6 +10,12 @@ from unittest import TestCase from sonic_py_common.general import getstatusoutput_noshell +import sys +if sys.version_info.major == 3: + from unittest import mock +else: + import mock + SKU = 'multi-npu-01' ASIC_SKU = 'multi-npu-asic' @@ -17,6 +23,7 @@ HOSTNAME = 'multi_npu_platform_01' DEVICE_TYPE = 'LeafRouter' +@mock.patch('swsssdk.util.read_from_file', mock.MagicMock(return_value='mock_password')) class TestMultiNpuCfgGen(TestCase): def setUp(self): @@ -45,6 +52,10 @@ def run_script(self, argument, check_stderr=True, output_file=None, validateYang if utils.PY3x: output = output.decode() + + # ignore safe error in unittest infra, since this file exists only in a runtime device + output = output.replace("ERROR:root:Failed to read from /etc/shadow_redis_dir/shadow_redis_admin, errno is [Errno 2] No such file or directory: '/etc/shadow_redis_dir/shadow_redis_admin'", '') + if output_file: with open(output_file, 'w') as f: f.write(output) @@ -538,12 +549,6 @@ def test_buffers_chassis_packet_lc_template(self): } ) - def test_bgpd_frr_frontendasic(self): - self.assertTrue(*self.run_frr_asic_case('bgpd/bgpd.conf.j2', 'bgpd_frr_frontend_asic.conf', "asic0", self.port_config[0])) - - def test_bgpd_frr_backendasic(self): - self.assertTrue(*self.run_frr_asic_case('bgpd/bgpd.conf.j2', 'bgpd_frr_backend_asic.conf', "asic3", self.port_config[3])) - def test_no_asic_in_graph(self): argument = ["-m", self.sample_graph, "-p", self.sample_no_asic_port_config, "-n", "asic4", "--var-json", "PORTCHANNEL"] output = json.loads(self.run_script(argument, check_stderr=False, validateYang=False))