Skip to content

Commit

Permalink
Issue 7488 docker labels (netdata#7770)
Browse files Browse the repository at this point in the history 
Improve the metadata detection for containers. The system_info structure has been updated to hold separate copies of OS_NAME, OS_ID, OS_ID_LIKE, OS_VERSION, OS_VERSION_ID and OS_DETECTION for both the container environment and the host. This new information is communicated through the /api/v1/info endpoint. For the streaming interface a partial copy of the info is carried until the stream protocol is upgraded. The anonymous_statistics script has been updated to carry the new data to Google Analytics. Some minor improvements have been made to OS-X / FreeBSD detection, and the detection of virtualization. The docs have been updated to explain how to pass the host environment to the docker container running Netdata.
  • Loading branch information
@amoss
amoss authored Jan 21, 2020
1 parent 434c9ba commit 49f6950
Show file tree
Hide file tree
Showing 8 changed files with 347 additions and 165 deletions.
37 changes: 25 additions & 12 deletions daemon/anonymous-statistics.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v')
# -------------------------------------------------------------------------------------------------
# send the anonymous statistics to GA
# https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters
# The maximum index for a cd parameter is 20 so we have effectively run out.
if [ -n "$(command -v curl 2>/dev/null)" ]; then
curl -X POST -Ss --max-time 2 \
--data "v=1" \
Expand All @@ -43,19 +44,25 @@ if [ -n "$(command -v curl 2>/dev/null)" ]; then
--data-urlencode "ec=${ACTION}" \
--data-urlencode "ea=${ACTION_RESULT}" \
--data-urlencode "el=${ACTION_DATA}" \
--data-urlencode "cd1=${NETDATA_SYSTEM_OS_NAME}" \
--data-urlencode "cd2=${NETDATA_SYSTEM_OS_ID}" \
--data-urlencode "cd3=${NETDATA_SYSTEM_OS_ID_LIKE}" \
--data-urlencode "cd4=${NETDATA_SYSTEM_OS_VERSION}" \
--data-urlencode "cd5=${NETDATA_SYSTEM_OS_VERSION_ID}" \
--data-urlencode "cd6=${NETDATA_SYSTEM_OS_DETECTION}" \
--data-urlencode "cd1=${NETDATA_HOST_OS_NAME}" \
--data-urlencode "cd2=${NETDATA_HOST_OS_ID}" \
--data-urlencode "cd3=${NETDATA_HOST_OS_ID_LIKE}" \
--data-urlencode "cd4=${NETDATA_HOST_OS_VERSION}" \
--data-urlencode "cd5=${NETDATA_HOST_OS_VERSION_ID}" \
--data-urlencode "cd6=${NETDATA_HOST_OS_DETECTION}" \
--data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \
--data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \
--data-urlencode "cd9=${NETDATA_SYSTEM_ARCHITECTURE}" \
--data-urlencode "cd10=${NETDATA_SYSTEM_VIRTUALIZATION}" \
--data-urlencode "cd11=${NETDATA_SYSTEM_VIRT_DETECTION}" \
--data-urlencode "cd12=${NETDATA_SYSTEM_CONTAINER}" \
--data-urlencode "cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}" \
--data-urlencode "cd14=${NETDATA_CONTAINER_OS_NAME}" \
--data-urlencode "cd15=${NETDATA_CONTAINER_OS_ID}" \
--data-urlencode "cd16=${NETDATA_CONTAINER_OS_ID_LIKE}" \
--data-urlencode "cd17=${NETDATA_CONTAINER_OS_VERSION}" \
--data-urlencode "cd18=${NETDATA_CONTAINER_OS_VERSION_ID}" \
--data-urlencode "cd19=${NETDATA_CONTAINER_OS_DETECTION}" \
"https://www.google-analytics.com/collect" >/dev/null 2>&1
else
wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\
Expand All @@ -72,18 +79,24 @@ else
&ec=${ACTION}\
&ea=${ACTION_RESULT}\
&el=${ACTION_DATA}\
&cd1=${NETDATA_SYSTEM_OS_NAME}\
&cd2=${NETDATA_SYSTEM_OS_ID}\
&cd3=${NETDATA_SYSTEM_OS_ID_LIKE}\
&cd4=${NETDATA_SYSTEM_OS_VERSION}\
&cd5=${NETDATA_SYSTEM_OS_VERSION_ID}\
&cd6=${NETDATA_SYSTEM_OS_DETECTION}\
&cd1=${NETDATA_HOST_OS_NAME}\
&cd2=${NETDATA_HOST_OS_ID}\
&cd3=${NETDATA_HOST_OS_ID_LIKE}\
&cd4=${NETDATA_HOST_OS_VERSION}\
&cd5=${NETDATA_HOST_OS_VERSION_ID}\
&cd6=${NETDATA_HOST_OS_DETECTION}\
&cd7=${NETDATA_SYSTEM_KERNEL_NAME}\
&cd8=${NETDATA_SYSTEM_KERNEL_VERSION}\
&cd9=${NETDATA_SYSTEM_ARCHITECTURE}\
&cd10=${NETDATA_SYSTEM_VIRTUALIZATION}\
&cd11=${NETDATA_SYSTEM_VIRT_DETECTION}\
&cd12=${NETDATA_SYSTEM_CONTAINER}\
&cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}\
&cd14=${NETDATA_CONTAINER_OS_NAME} \
&cd15=${NETDATA_CONTAINER_OS_ID} \
&cd16=${NETDATA_CONTAINER_OS_ID_LIKE} \
&cd17=${NETDATA_CONTAINER_OS_VERSION} \
&cd18=${NETDATA_CONTAINER_OS_VERSION_ID} \
&cd19=${NETDATA_CONTAINER_OS_DETECTION} \
" > /dev/null 2>&1
fi
36 changes: 21 additions & 15 deletions daemon/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,12 @@ static int load_netdata_conf(char *filename, char overwrite_used) {
return ret;
}

// coverity[ +tainted_string_sanitize_content : arg-0 ]
inline void coverity_remove_taint(char *s)
{
(void)s;
}

int get_system_info(struct rrdhost_system_info *system_info) {
char *script;
script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("system-info.sh") + 2));
Expand All @@ -747,27 +753,27 @@ int get_system_info(struct rrdhost_system_info *system_info) {

FILE *fp = mypopen(script, &command_pid);
if(fp) {
char buffer[200 + 1];
while (fgets(buffer, 200, fp) != NULL) {
char *name=buffer;
char *value=buffer;
char line[200 + 1];
// Removed the double strlens, if the Coverity tainted string warning reappears I'll revert.
// One time init code, but I'm curious about the warning...
while (fgets(line, 200, fp) != NULL) {
char *value=line;
while (*value && *value != '=') value++;
if (*value=='=') {
*value='\0';
value++;
if (strlen(value)>1) {
char *newline = value + strlen(value) - 1;
(*newline) = '\0';
}
char n[51], v[101];
snprintfz(n, 50,"%s",name);
snprintfz(v, 100,"%s",value);
if(unlikely(rrdhost_set_system_info_variable(system_info, n, v))) {
info("Unexpected environment variable %s=%s", n, v);
char *end = value;
while (*end && *end != '\n') end++;
*end = '\0'; // Overwrite newline if present
coverity_remove_taint(line); // I/O is controlled result of system_info.sh - not tainted
coverity_remove_taint(value);

if(unlikely(rrdhost_set_system_info_variable(system_info, line, value))) {
info("Unexpected environment variable %s=%s", line, value);
}
else {
info("%s=%s", n, v);
setenv(n, v, 1);
info("%s=%s", line, value);
setenv(line, value, 1);
}
}
}
Expand Down
245 changes: 157 additions & 88 deletions daemon/system-info.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,111 +8,180 @@ KERNEL_VERSION="$(uname -r)"
ARCHITECTURE="$(uname -m)"

# -------------------------------------------------------------------------------------------------
# detect the operating system
# detect the virtualization

OS_DETECTION="unknown"
NAME="unknown"
VERSION="unknown"
VERSION_ID="unknown"
ID="unknown"
ID_LIKE="unknown"
if [ -z "${VIRTUALIZATION}" ]; then
VIRTUALIZATION="unknown"
VIRT_DETECTION="none"

if [ "${KERNEL_NAME}" = "Darwin" ]; then
# Mac OS
OIFS="$IFS"
IFS=$'\n'
set $(sw_vers) > /dev/null
NAME=$(echo $1 | tr "\n\t" ' ' | sed -e 's/ProductName:[ ]*//' -e 's/[ ]*$//')
VERSION=$(echo $2 | tr "\n\t" ' ' | sed -e 's/ProductVersion:[ ]*//' -e 's/[ ]*$//')
ID="mac"
ID_LIKE="mac"
OS_DETECTION="sw_vers"
IFS="$OIFS"
if [ -n "$(command -v systemd-detect-virt 2>/dev/null)" ]; then
VIRTUALIZATION="$(systemd-detect-virt -v)"
VIRT_DETECTION="systemd-detect-virt"
CONTAINER="$(systemd-detect-virt -c)"
CONT_DETECTION="systemd-detect-virt"
else
if grep -q "^flags.*hypervisor" /proc/cpuinfo 2>/dev/null; then
VIRTUALIZATION="hypervisor"
VIRT_DETECTION="/proc/cpuinfo"
elif [ -n "$(command -v dmidecode)" ]; then
# Virtualization detection from https://unix.stackexchange.com/questions/89714/easy-way-to-determine-virtualization-technology
# This only works as root
if dmidecode -s system-product-name 2>/dev/null | grep -q "VMware\|Virtual\|KVM\|Bochs"; then
VIRTUALIZATION="$(dmidecode -s system-product-name)"
VIRT_DETECTION="dmidecode"
fi
fi
fi
else
if [ -f "/etc/os-release" ]; then
OS_DETECTION="/etc/os-release"
eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" </etc/os-release)"
fi

if [ "${NAME}" = "unknown" ] || [ "${VERSION}" = "unknown" ] || [ "${ID}" = "unknown" ]; then
if [ -f "/etc/lsb-release" ]; then
if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="/etc/lsb-release"; else OS_DETECTION="Mixed"; fi
DISTRIB_ID="unknown"
DISTRIB_RELEASE="unknown"
DISTRIB_CODENAME="unknown"
eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" </etc/lsb-release)"
if [ "${NAME}" = "unknown" ]; then NAME="${DISTRIB_ID}"; fi
if [ "${VERSION}" = "unknown" ]; then VERSION="${DISTRIB_RELEASE}"; fi
if [ "${ID}" = "unknown" ]; then ID="${DISTRIB_CODENAME}"; fi
fi
if [ -n "$(command -v lsb_release 2>/dev/null)" ]; then
if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="lsb_release"; else OS_DETECTION="Mixed"; fi
if [ "${NAME}" = "unknown" ]; then NAME="$(lsb_release -is 2>/dev/null)"; fi
if [ "${VERSION}" = "unknown" ]; then VERSION="$(lsb_release -rs 2>/dev/null)"; fi
if [ "${ID}" = "unknown" ]; then ID="$(lsb_release -cs 2>/dev/null)"; fi
fi
fi
# Passed from outside - probably in docker run
VIRT_DETECTION="provided"
fi

# -------------------------------------------------------------------------------------------------
# detect the virtualization
# detect containers with heuristics

VIRTUALIZATION="unknown"
VIRT_DETECTION="none"
CONTAINER="unknown"
CONT_DETECTION="none"

if [ -n "$(command -v systemd-detect-virt 2>/dev/null)" ]; then
VIRTUALIZATION="$(systemd-detect-virt -v)"
VIRT_DETECTION="systemd-detect-virt"
CONTAINER="$(systemd-detect-virt -c)"
CONT_DETECTION="systemd-detect-virt"
else
if grep -q "^flags.*hypervisor" /proc/cpuinfo 2>/dev/null; then
VIRTUALIZATION="hypervisor"
VIRT_DETECTION="/proc/cpuinfo"
fi
if [ "${CONTAINER}" = "unknown" ]; then
if [ -f /proc/1/sched ] ; then
IFS='(, ' read -r process _ </proc/1/sched
if [ "${process}" = "netdata" ]; then
CONTAINER="container"
CONT_DETECTION="process"
fi
fi
# ubuntu and debian supply /bin/running-in-container
# https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container
if /bin/running-in-container >/dev/null 2>&1; then
CONTAINER="container"
CONT_DETECTION="/bin/running-in-container"
fi

# lxc sets environment variable 'container'
#shellcheck disable=SC2154
if [ -n "${container}" ]; then
CONTAINER="lxc"
CONT_DETECTION="containerenv"
fi

# docker creates /.dockerenv
# http://stackoverflow.com/a/25518345
if [ -f "/.dockerenv" ]; then
CONTAINER="docker"
CONT_DETECTION="dockerenv"
fi

fi

# -------------------------------------------------------------------------------------------------
# detect containers with heuristics
# detect the operating system

# Initially assume all OS detection values are for a container, these are moved later if we are bare-metal

CONTAINER_OS_DETECTION="unknown"
CONTAINER_NAME="unknown"
CONTAINER_VERSION="unknown"
CONTAINER_VERSION_ID="unknown"
CONTAINER_ID="unknown"
CONTAINER_ID_LIKE="unknown"

if [ "${KERNEL_NAME}" = "Darwin" ]; then
CONTAINER_ID=$(sw_vers -productName)
CONTAINER_ID_LIKE="mac"
CONTAINER_NAME="mac"
CONTAINER_VERSION=$(sw_vers -productVersion)
CONTAINER_OS_DETECTION="sw_vers"
elif [ "${KERNEL_NAME}" = "FreeBSD" ]; then
CONTAINER_ID="FreeBSD"
CONTAINER_ID_LIKE="FreeBSD"
CONTAINER_NAME="FreeBSD"
CONTAINER_OS_DETECTION="uname"
CONTAINER_VERSION=$(uname -r)
KERNEL_VERSION=$(uname -K)
else
if [ -f "/etc/os-release" ]; then
eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" </etc/os-release | sed 's/^/CONTAINER_/')"
CONTAINER_OS_DETECTION="/etc/os-release"
fi

if [ "${NAME}" = "unknown" ] || [ "${VERSION}" = "unknown" ] || [ "${ID}" = "unknown" ]; then
if [ -f "/etc/lsb-release" ]; then
if [ "${OS_DETECTION}" = "unknown" ]; then
CONTAINER_OS_DETECTION="/etc/lsb-release"
else
CONTAINER_OS_DETECTION="Mixed"
fi
DISTRIB_ID="unknown"
DISTRIB_RELEASE="unknown"
DISTRIB_CODENAME="unknown"
eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" </etc/lsb-release)"
if [ "${NAME}" = "unknown" ]; then CONTAINER_NAME="${DISTRIB_ID}"; fi
if [ "${VERSION}" = "unknown" ]; then CONTAINER_VERSION="${DISTRIB_RELEASE}"; fi
if [ "${ID}" = "unknown" ]; then CONTAINER_ID="${DISTRIB_CODENAME}"; fi
fi
if [ -n "$(command -v lsb_release 2>/dev/null)" ]; then
if [ "${OS_DETECTION}" = "unknown" ]; then
CONTAINER_OS_DETECTION="lsb_release"
else
CONTAINER_OS_DETECTION="Mixed"
fi
if [ "${NAME}" = "unknown" ]; then CONTAINER_NAME="$(lsb_release -is 2>/dev/null)"; fi
if [ "${VERSION}" = "unknown" ]; then CONTAINER_VERSION="$(lsb_release -rs 2>/dev/null)"; fi
if [ "${ID}" = "unknown" ]; then CONTAINER_ID="$(lsb_release -cs 2>/dev/null)"; fi
fi
fi
fi

# If Netdata is not running in a container then use the local detection as the host
HOST_OS_DETECTION="unknown"
HOST_NAME="unknown"
HOST_VERSION="unknown"
HOST_VERSION_ID="unknown"
HOST_ID="unknown"
HOST_ID_LIKE="unknown"
if [ "${CONTAINER}" = "unknown" ]; then
if [ -f /proc/1/sched ] ; then
IFS='(, ' read -r process _ </proc/1/sched
if [ "${process}" = "netdata" ]; then
CONTAINER="container"
CONT_DETECTION="process"
fi
fi
# ubuntu and debian supply /bin/running-in-container
# https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container
if /bin/running-in-container >/dev/null 2>&1; then
CONTAINER="container"
CONT_DETECTION="/bin/running-in-container"
fi

# lxc sets environment variable 'container'
#shellcheck disable=SC2154
if [ -n "${container}" ]; then
CONTAINER="lxc"
CONT_DETECTION="containerenv"
fi

# docker creates /.dockerenv
# http://stackoverflow.com/a/25518345
if [ -f "/.dockerenv" ]; then
CONTAINER="docker"
CONT_DETECTION="dockerenv"
fi
for v in NAME ID ID_LIKE VERSION VERSION_ID OS_DETECTION; do
eval "HOST_$v=\$CONTAINER_$v; CONTAINER_$v=none"
done
else
# Otherwise try and use a user-supplied bind-mount into the container to resolve the host details
if [ -e "/host/etc/os-release" ]; then
OS_DETECTION="/etc/os-release"
eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" </host/etc/os-release | sed 's/^/HOST_/')"
HOST_OS_DETECTION="/host/etc/os-release"
fi
if [ "${HOST_NAME}" = "unknown" ] || [ "${HOST_VERSION}" = "unknown" ] || [ "${HOST_ID}" = "unknown" ]; then
if [ -f "/host/etc/lsb-release" ]; then
if [ "${HOST_OS_DETECTION}" = "unknown" ]; then
HOST_OS_DETECTION="/etc/lsb-release"
else
HOST_OS_DETECTION="Mixed"
fi
DISTRIB_ID="unknown"
DISTRIB_RELEASE="unknown"
DISTRIB_CODENAME="unknown"
eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" </etc/lsb-release)"
if [ "${HOST_NAME}" = "unknown" ]; then HOST_NAME="${DISTRIB_ID}"; fi
if [ "${HOST_VERSION}" = "unknown" ]; then HOST_VERSION="${DISTRIB_RELEASE}"; fi
if [ "${HOST_ID}" = "unknown" ]; then HOST_ID="${DISTRIB_CODENAME}"; fi
fi
fi
fi

echo "NETDATA_SYSTEM_OS_NAME=${NAME}"
echo "NETDATA_SYSTEM_OS_ID=${ID}"
echo "NETDATA_SYSTEM_OS_ID_LIKE=${ID_LIKE}"
echo "NETDATA_SYSTEM_OS_VERSION=${VERSION}"
echo "NETDATA_SYSTEM_OS_VERSION_ID=${VERSION_ID}"
echo "NETDATA_SYSTEM_OS_DETECTION=${OS_DETECTION}"

echo "NETDATA_CONTAINER_OS_NAME=${CONTAINER_NAME}"
echo "NETDATA_CONTAINER_OS_ID=${CONTAINER_ID}"
echo "NETDATA_CONTAINER_OS_ID_LIKE=${CONTAINER_ID_LIKE}"
echo "NETDATA_CONTAINER_OS_VERSION=${CONTAINER_VERSION}"
echo "NETDATA_CONTAINER_OS_VERSION_ID=${CONTAINER_VERSION_ID}"
echo "NETDATA_CONTAINER_OS_DETECTION=${CONTAINER_OS_DETECTION}"
echo "NETDATA_HOST_OS_NAME=${HOST_NAME}"
echo "NETDATA_HOST_OS_ID=${HOST_ID}"
echo "NETDATA_HOST_OS_ID_LIKE=${HOST_ID_LIKE}"
echo "NETDATA_HOST_OS_VERSION=${HOST_VERSION}"
echo "NETDATA_HOST_OS_VERSION_ID=${HOST_VERSION_ID}"
echo "NETDATA_HOST_OS_DETECTION=${HOST_OS_DETECTION}"
echo "NETDATA_SYSTEM_KERNEL_NAME=${KERNEL_NAME}"
echo "NETDATA_SYSTEM_KERNEL_VERSION=${KERNEL_VERSION}"
echo "NETDATA_SYSTEM_ARCHITECTURE=${ARCHITECTURE}"
Expand Down
Loading

0 comments on commit 49f6950

Please sign in to comment.