From 989b1ab14f76f2191c1752f1e1f9089e98597d9d Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Thu, 1 Jun 2017 10:25:05 -0700 Subject: [PATCH] Move calico/node from projectcalico/calicoctl --- .gitignore | 25 + Dockerfile | 39 + Makefile | 502 +++++++++++ allocateipip/allocate_ipip_addr.go | 145 ++++ allocateipip/allocateipip_suite_test.go | 19 + calico_test/Dockerfile.calico_test | 55 ++ calico_test/requirements.txt | 7 + calicoclient/calicoclient.go | 27 + calicoclient/calicoclient_suite_test.go | 19 + .../etc/calico/confd/conf.d/bird.toml.toml | 8 + .../etc/calico/confd/conf.d/bird6.toml.toml | 8 + .../etc/calico/confd/conf.d/bird6_ipam.toml | 8 + .../etc/calico/confd/conf.d/bird_ipam.toml | 8 + .../calico/confd/conf.d/custom_filters.toml | 8 + .../calico/confd/conf.d/custom_filters6.toml | 8 + .../etc/calico/confd/conf.d/tunl-ip.toml | 8 + filesystem/etc/calico/confd/config/.gitkeep | 0 .../etc/calico/confd/templates/README.md | 47 ++ .../confd/templates/bird.cfg.mesh.template | 99 +++ .../confd/templates/bird.cfg.no-mesh.template | 84 ++ .../calico/confd/templates/bird.toml.template | 10 + .../confd/templates/bird6.cfg.mesh.template | 103 +++ .../templates/bird6.cfg.no-mesh.template | 87 ++ .../confd/templates/bird6.toml.template | 10 + .../confd/templates/bird6_aggr.toml.template | 8 + .../confd/templates/bird6_ipam.cfg.template | 11 + .../confd/templates/bird_aggr.cfg.template | 22 + .../confd/templates/bird_aggr.toml.template | 8 + .../confd/templates/bird_ipam.cfg.template | 32 + .../confd/templates/bird_ipam.toml.template | 9 + .../templates/custom_filters.cfg.template | 7 + .../templates/custom_filters6.cfg.template | 7 + .../calico/confd/templates/tunl-ip.template | 7 + filesystem/etc/calico/felix.cfg | 5 + filesystem/etc/rc.local | 103 +++ filesystem/etc/service/available/bird/log/run | 5 + filesystem/etc/service/available/bird/run | 3 + .../etc/service/available/bird6/log/run | 5 + filesystem/etc/service/available/bird6/run | 3 + .../available/calico-bgp-daemon/log/run | 4 + .../service/available/calico-bgp-daemon/run | 3 + .../etc/service/available/confd/log/run | 4 + filesystem/etc/service/available/confd/run | 14 + .../etc/service/available/felix/log/run | 4 + filesystem/etc/service/available/felix/run | 18 + .../etc/service/available/libnetwork/log/run | 4 + .../etc/service/available/libnetwork/run | 3 + filesystem/sbin/restart-calico-confd | 2 + filesystem/sbin/start_runit | 16 + filesystem/sbin/versions | 10 + glide.lock | 325 ++++++++ glide.yaml | 18 + .../autodetection/autodetection_suite_test.go | 19 + startup/autodetection/filtered.go | 51 ++ startup/autodetection/filtered_test.go | 36 + startup/autodetection/interfaces.go | 104 +++ startup/autodetection/reachaddr.go | 66 ++ startup/startup.go | 779 ++++++++++++++++++ startup/startup_suite_test.go | 19 + startup/startup_test.go | 268 ++++++ tests/__init__.py | 0 tests/st/__init__.py | 0 tests/st/bgp/__init__.py | 0 tests/st/bgp/peer.py | 31 + tests/st/bgp/test_backends.py | 73 ++ tests/st/bgp/test_global_config.py | 92 +++ tests/st/bgp/test_global_peers.py | 84 ++ tests/st/bgp/test_ipip.py | 419 ++++++++++ tests/st/bgp/test_node_peers.py | 81 ++ tests/st/bgp/test_node_status_resilience.py | 147 ++++ tests/st/bgp/test_route_reflector_cluster.py | 86 ++ tests/st/bgp/test_single_route_reflector.py | 78 ++ tests/st/bgp/test_update_ip_addr.py | 69 ++ tests/st/calicoctl/__init__.py | 0 tests/st/calicoctl/test_autodetection.py | 103 +++ tests/st/calicoctl/test_default_pools.py | 161 ++++ tests/st/calicoctl/test_node_checksystem.py | 31 + tests/st/calicoctl/test_node_diags.py | 38 + tests/st/calicoctl/test_node_run.py | 31 + tests/st/calicoctl/test_node_status.py | 49 ++ tests/st/ipam/__init__.py | 0 tests/st/ipam/test_ipam.py | 225 +++++ tests/st/libnetwork/test_labeling.py | 177 ++++ tests/st/policy/__init__.py | 0 tests/st/policy/test_felix_gateway.py | 493 +++++++++++ tests/st/policy/test_profile.py | 327 ++++++++ tests/st/ssl-config/ca-config.json | 13 + tests/st/ssl-config/ca-csr.json | 16 + tests/st/ssl-config/req-csr.json | 19 + tests/st/test_base.py | 305 +++++++ tests/st/utils/__init__.py | 0 tests/st/utils/constants.py | 20 + tests/st/utils/docker_host.py | 599 ++++++++++++++ tests/st/utils/exceptions.py | 42 + tests/st/utils/log_analyzer.py | 355 ++++++++ tests/st/utils/network.py | 93 +++ tests/st/utils/route_reflector.py | 120 +++ tests/st/utils/utils.py | 387 +++++++++ tests/st/utils/workload.py | 360 ++++++++ tests/ut/__init__.py | 0 tests/ut/test_log_parsing.py | 142 ++++ workload/Dockerfile | 8 + workload/responder.py | 106 +++ workload/tcpping.sh | 2 + workload/udpping.sh | 2 + 105 files changed, 8720 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 allocateipip/allocate_ipip_addr.go create mode 100644 allocateipip/allocateipip_suite_test.go create mode 100644 calico_test/Dockerfile.calico_test create mode 100644 calico_test/requirements.txt create mode 100644 calicoclient/calicoclient.go create mode 100644 calicoclient/calicoclient_suite_test.go create mode 100644 filesystem/etc/calico/confd/conf.d/bird.toml.toml create mode 100644 filesystem/etc/calico/confd/conf.d/bird6.toml.toml create mode 100644 filesystem/etc/calico/confd/conf.d/bird6_ipam.toml create mode 100644 filesystem/etc/calico/confd/conf.d/bird_ipam.toml create mode 100644 filesystem/etc/calico/confd/conf.d/custom_filters.toml create mode 100644 filesystem/etc/calico/confd/conf.d/custom_filters6.toml create mode 100644 filesystem/etc/calico/confd/conf.d/tunl-ip.toml create mode 100644 filesystem/etc/calico/confd/config/.gitkeep create mode 100644 filesystem/etc/calico/confd/templates/README.md create mode 100644 filesystem/etc/calico/confd/templates/bird.cfg.mesh.template create mode 100644 filesystem/etc/calico/confd/templates/bird.cfg.no-mesh.template create mode 100644 filesystem/etc/calico/confd/templates/bird.toml.template create mode 100644 filesystem/etc/calico/confd/templates/bird6.cfg.mesh.template create mode 100644 filesystem/etc/calico/confd/templates/bird6.cfg.no-mesh.template create mode 100644 filesystem/etc/calico/confd/templates/bird6.toml.template create mode 100644 filesystem/etc/calico/confd/templates/bird6_aggr.toml.template create mode 100644 filesystem/etc/calico/confd/templates/bird6_ipam.cfg.template create mode 100644 filesystem/etc/calico/confd/templates/bird_aggr.cfg.template create mode 100644 filesystem/etc/calico/confd/templates/bird_aggr.toml.template create mode 100644 filesystem/etc/calico/confd/templates/bird_ipam.cfg.template create mode 100644 filesystem/etc/calico/confd/templates/bird_ipam.toml.template create mode 100644 filesystem/etc/calico/confd/templates/custom_filters.cfg.template create mode 100644 filesystem/etc/calico/confd/templates/custom_filters6.cfg.template create mode 100644 filesystem/etc/calico/confd/templates/tunl-ip.template create mode 100644 filesystem/etc/calico/felix.cfg create mode 100755 filesystem/etc/rc.local create mode 100755 filesystem/etc/service/available/bird/log/run create mode 100755 filesystem/etc/service/available/bird/run create mode 100755 filesystem/etc/service/available/bird6/log/run create mode 100755 filesystem/etc/service/available/bird6/run create mode 100755 filesystem/etc/service/available/calico-bgp-daemon/log/run create mode 100755 filesystem/etc/service/available/calico-bgp-daemon/run create mode 100755 filesystem/etc/service/available/confd/log/run create mode 100755 filesystem/etc/service/available/confd/run create mode 100755 filesystem/etc/service/available/felix/log/run create mode 100755 filesystem/etc/service/available/felix/run create mode 100755 filesystem/etc/service/available/libnetwork/log/run create mode 100755 filesystem/etc/service/available/libnetwork/run create mode 100755 filesystem/sbin/restart-calico-confd create mode 100755 filesystem/sbin/start_runit create mode 100755 filesystem/sbin/versions create mode 100644 glide.lock create mode 100644 glide.yaml create mode 100644 startup/autodetection/autodetection_suite_test.go create mode 100644 startup/autodetection/filtered.go create mode 100644 startup/autodetection/filtered_test.go create mode 100644 startup/autodetection/interfaces.go create mode 100644 startup/autodetection/reachaddr.go create mode 100644 startup/startup.go create mode 100644 startup/startup_suite_test.go create mode 100644 startup/startup_test.go create mode 100644 tests/__init__.py create mode 100644 tests/st/__init__.py create mode 100644 tests/st/bgp/__init__.py create mode 100644 tests/st/bgp/peer.py create mode 100644 tests/st/bgp/test_backends.py create mode 100644 tests/st/bgp/test_global_config.py create mode 100644 tests/st/bgp/test_global_peers.py create mode 100644 tests/st/bgp/test_ipip.py create mode 100644 tests/st/bgp/test_node_peers.py create mode 100644 tests/st/bgp/test_node_status_resilience.py create mode 100644 tests/st/bgp/test_route_reflector_cluster.py create mode 100644 tests/st/bgp/test_single_route_reflector.py create mode 100644 tests/st/bgp/test_update_ip_addr.py create mode 100644 tests/st/calicoctl/__init__.py create mode 100644 tests/st/calicoctl/test_autodetection.py create mode 100644 tests/st/calicoctl/test_default_pools.py create mode 100644 tests/st/calicoctl/test_node_checksystem.py create mode 100644 tests/st/calicoctl/test_node_diags.py create mode 100644 tests/st/calicoctl/test_node_run.py create mode 100644 tests/st/calicoctl/test_node_status.py create mode 100644 tests/st/ipam/__init__.py create mode 100644 tests/st/ipam/test_ipam.py create mode 100644 tests/st/libnetwork/test_labeling.py create mode 100644 tests/st/policy/__init__.py create mode 100644 tests/st/policy/test_felix_gateway.py create mode 100644 tests/st/policy/test_profile.py create mode 100644 tests/st/ssl-config/ca-config.json create mode 100644 tests/st/ssl-config/ca-csr.json create mode 100644 tests/st/ssl-config/req-csr.json create mode 100644 tests/st/test_base.py create mode 100644 tests/st/utils/__init__.py create mode 100644 tests/st/utils/constants.py create mode 100644 tests/st/utils/docker_host.py create mode 100644 tests/st/utils/exceptions.py create mode 100644 tests/st/utils/log_analyzer.py create mode 100644 tests/st/utils/network.py create mode 100644 tests/st/utils/route_reflector.py create mode 100644 tests/st/utils/utils.py create mode 100644 tests/st/utils/workload.py create mode 100644 tests/ut/__init__.py create mode 100644 tests/ut/test_log_parsing.py create mode 100644 workload/Dockerfile create mode 100644 workload/responder.py create mode 100644 workload/tcpping.sh create mode 100644 workload/udpping.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..5aa1876d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +.go-pkg-cache +build +dist +venv +env +certs +.idea +*.pyc +*~ +.coverage +cover +*.tar +*.created +docker +**/*.sw[pon] +birdcl +.vagrant +gobgp +calicoctl/calicoctl +bin +release +*.coverprofile +vendor +nosetests.xml +testfile.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..74b938dde --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM alpine +MAINTAINER Tom Denham + +# Set the minimum Docker API version required for libnetwork. +ENV DOCKER_API_VERSION 1.21 + +# Download and install glibc for use by non-static binaries that require it. +RUN apk --no-cache add wget ca-certificates libgcc && \ + wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk && \ + wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-bin-2.23-r3.apk && \ + apk add glibc-2.23-r3.apk glibc-bin-2.23-r3.apk && \ + /usr/glibc-compat/sbin/ldconfig /lib /usr/glibc/usr/lib && \ + apk del wget && \ + rm -f glibc-2.23-r3.apk glibc-bin-2.23-r3.apk + +# Install runit from the community repository, as its not yet available in global +RUN apk add --no-cache --repository "http://alpine.gliderlabs.com/alpine/edge/community" runit + +# Install remaining runtime deps required for felix from the global repository +RUN apk add --no-cache ip6tables ipset iputils iproute2 conntrack-tools + +# Copy in the filesystem - this contains felix, bird, gobgp etc... +COPY filesystem / + +CMD ["start_runit"] diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..97d735818 --- /dev/null +++ b/Makefile @@ -0,0 +1,502 @@ +############################################################################### +# Versions: +RELEASE_STREAM?=v2.3 +CALICO_NODE_DIR=$(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +VERSIONS_FILE?=$(CALICO_NODE_DIR)/../_data/versions.yml +YAML_CMD?=$(shell which yaml || echo docker run -i calico/go-build yaml) +# For local builds this can be made faster by running "go get github.com/mikefarah/yaml" and changing YAML_CMD to "yaml" +CALICO_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].title') +BIRD_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].components.calico-bird.version') +GOBGPD_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].components.calico-bgp-daemon.version') +FELIX_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].components.felix.version') +CALICOCTL_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].components.calicoctl.version') +LIBNETWORK_PLUGIN_VER := $(shell cat $(VERSIONS_FILE) | $(YAML_CMD) read - '"$(RELEASE_STREAM)".[0].components.libnetwork-plugin.version') +# TODO - Why isn't confd in versions.yaml +CONFD_VER := v0.12.1-calico0.1.0 + +SYSTEMTEST_CONTAINER_VER := latest +GO_BUILD_VER:=latest +# we can use "custom" build image and test image name +SYSTEMTEST_CONTAINER?=calico/test:$(SYSTEMTEST_CONTAINER_VER) + +# Ensure that the dist directory is always created +MAKE_SURE_BIN_EXIST := $(shell mkdir -p dist) + +############################################################################### +# URL for Calico binaries +# confd binary +CONFD_URL?=https://github.com/projectcalico/confd/releases/download/$(CONFD_VER)/confd +# bird binaries +BIRD_URL?=https://github.com/projectcalico/calico-bird/releases/download/$(BIRD_VER)/bird +BIRD6_URL?=https://github.com/projectcalico/calico-bird/releases/download/$(BIRD_VER)/bird6 +BIRDCL_URL?=https://github.com/projectcalico/calico-bird/releases/download/$(BIRD_VER)/birdcl +CALICO_BGP_DAEMON_URL?=https://github.com/projectcalico/calico-bgp-daemon/releases/download/$(GOBGPD_VER)/calico-bgp-daemon +GOBGP_URL?=https://github.com/projectcalico/calico-bgp-daemon/releases/download/$(GOBGPD_VER)/gobgp + +############################################################################### +# calico/node build. Contains the following areas +# - Populate the calico_node/filesystem +# - Build the container itself +############################################################################### +NODE_CONTAINER_DIR=. +NODE_CONTAINER_NAME?=calico/node +NODE_CONTAINER_FILES=$(shell find $(NODE_CONTAINER_DIR)/filesystem -type f) +NODE_CONTAINER_CREATED=$(NODE_CONTAINER_DIR)/.calico_node.created +NODE_CONTAINER_BIN_DIR=$(NODE_CONTAINER_DIR)/filesystem/bin +NODE_CONTAINER_BINARIES=startup allocate-ipip-addr calico-felix bird calico-bgp-daemon confd libnetwork-plugin +FELIX_CONTAINER_NAME?=calico/felix:$(FELIX_VER) +LIBNETWORK_PLUGIN_CONTAINER_NAME?=calico/libnetwork-plugin:$(LIBNETWORK_PLUGIN_VER) + +STARTUP_DIR=$(NODE_CONTAINER_DIR)/startup +STARTUP_FILES=$(shell find $(STARTUP_DIR) -name '*.go') +ALLOCATE_IPIP_DIR=$(NODE_CONTAINER_DIR)/allocateipip +ALLOCATE_IPIP_FILES=$(shell find $(ALLOCATE_IPIP_DIR) -name '*.go') + +TEST_CONTAINER_NAME?=calico/test:latest +TEST_CONTAINER_FILES=$(shell find tests/ -type f ! -name '*.created') + +CALICO_BUILD?=calico/go-build:$(GO_BUILD_VER) +LOCAL_USER_ID?=$(shell id -u $$USER) + +# TODO - This should be changed +PACKAGE_NAME?=github.com/projectcalico/calico/calico_node + +LIBCALICOGO_PATH?=none + +# Use this to populate the vendor directory after checking out the repository. +# To update upstream dependencies, delete the glide.lock file first. +vendor: glide.yaml + # Ensure that the glide cache directory exists. + mkdir -p $(HOME)/.glide + + # To build without Docker just run "glide install -strip-vendor" + if [ "$(LIBCALICOGO_PATH)" != "none" ]; then \ + EXTRA_DOCKER_BIND="-v $(LIBCALICOGO_PATH):/go/src/github.com/projectcalico/libcalico-go:ro"; \ + fi; \ + docker run --rm \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw $$EXTRA_DOCKER_BIND \ + -v $(HOME)/.glide:/home/user/.glide:rw \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + $(CALICO_BUILD) /bin/sh -c ' \ + cd /go/src/$(PACKAGE_NAME) && \ + glide install -strip-vendor' + + +# Download calicoctl v1.0.2 from releases. Used for STs (testing pre/post v1.1.0 data model) +dist/calicoctl-v1.0.2: + wget https://github.com/projectcalico/calicoctl/releases/download/v1.0.2/calicoctl -O dist/calicoctl-v1.0.2 + chmod +x dist/calicoctl-v1.0.2 + +dist/calicoctl: + wget https://github.com/projectcalico/calicoctl/releases/download/$(CALICOCTL_VER)/calicoctl -O dist/calicoctl + chmod +x dist/calicoctl + +test_image: calico_test.created ## Create the calico/test image + +calico_test.created: $(TEST_CONTAINER_FILES) + cd calico_test && docker build -f Dockerfile.calico_test -t $(TEST_CONTAINER_NAME) . + touch calico_test.created + +$(NODE_CONTAINER_NAME): $(NODE_CONTAINER_CREATED) ## Create the calico/node image + +calico-node.tar: $(NODE_CONTAINER_CREATED) + docker save --output $@ $(NODE_CONTAINER_NAME) + +# Build ACI (the APPC image file format) of calico/node. +# Requires docker2aci installed on host: https://github.com/appc/docker2aci +calico-node-latest.aci: calico-node.tar + docker2aci $< + +# Build calico/node docker image - explicitly depend on the container binaries. +$(NODE_CONTAINER_CREATED): $(NODE_CONTAINER_DIR)/Dockerfile $(NODE_CONTAINER_FILES) $(addprefix $(NODE_CONTAINER_BIN_DIR)/,$(NODE_CONTAINER_BINARIES)) + docker build --pull -t $(NODE_CONTAINER_NAME) $(NODE_CONTAINER_DIR) + touch $@ + +# Get felix binaries +.PHONY: update-felix +$(NODE_CONTAINER_BIN_DIR)/calico-felix update-felix: + -docker rm -f calico-felix + # Latest felix binaries are stored in automated builds of calico/felix. + # To get them, we create (but don't start) a container from that image. + docker create --name calico-felix $(FELIX_CONTAINER_NAME) + # Then we copy the files out of the container. Since docker preserves + # mtimes on its copy, check the file really did appear, then touch it + # to make sure that downstream targets get rebuilt. + docker cp calico-felix:/code/. $(NODE_CONTAINER_BIN_DIR) && \ + test -e $(NODE_CONTAINER_BIN_DIR)/calico-felix && \ + touch $(NODE_CONTAINER_BIN_DIR)/calico-felix + -docker rm -f calico-felix + +# Get libnetwork-plugin binaries +$(NODE_CONTAINER_BIN_DIR)/libnetwork-plugin: + -docker rm -f calico-$(@F) + # Latest libnetwork-plugin binaries are stored in automated builds of calico/libnetwork-plugin. + # To get them, we pull that image, then copy the binaries out to our host + docker create --name calico-$(@F) $(LIBNETWORK_PLUGIN_CONTAINER_NAME) + docker cp calico-$(@F):/$(@F) $(@D) + -docker rm -f calico-$(@F) + +# Get the confd binary +$(NODE_CONTAINER_BIN_DIR)/confd: + $(CURL) -L $(CONFD_URL) -o $@ + chmod +x $@ + +# Get the calico-bgp-daemon binary +$(NODE_CONTAINER_BIN_DIR)/calico-bgp-daemon: + $(CURL) -L $(GOBGP_URL) -o $(@D)/gobgp + $(CURL) -L $(CALICO_BGP_DAEMON_URL) -o $@ + chmod +x $(@D)/* + +# Get bird binaries +$(NODE_CONTAINER_BIN_DIR)/bird: + # This make target actually downloads the bird6 and birdcl binaries too + # Copy patched BIRD daemon with tunnel support. + $(CURL) -L $(BIRD6_URL) -o $(@D)/bird6 + $(CURL) -L $(BIRDCL_URL) -o $(@D)/birdcl + $(CURL) -L $(BIRD_URL) -o $@ + chmod +x $(@D)/* + +$(NODE_CONTAINER_BIN_DIR)/startup: dist/startup + mkdir -p $(NODE_CONTAINER_BIN_DIR) + cp dist/startup $(NODE_CONTAINER_BIN_DIR)/startup + +$(NODE_CONTAINER_BIN_DIR)/allocate-ipip-addr: dist/allocate-ipip-addr + mkdir -p $(NODE_CONTAINER_BIN_DIR) + cp dist/allocate-ipip-addr $(NODE_CONTAINER_BIN_DIR)/allocate-ipip-addr + +## Build startup.go +.PHONY: startup +startup: + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -i -o dist/startup $(LDFLAGS) startup/startup.go + +dist/startup: $(STARTUP_FILES) vendor + mkdir -p dist + mkdir -p .go-pkg-cache + docker run --rm \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + -v $(CURDIR)/.go-pkg-cache:/go/pkg/:rw \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):ro \ + -v $(CURDIR)/dist:/go/src/$(PACKAGE_NAME)/dist \ + -v $(VERSIONS_FILE):/versions.yaml:ro \ + -e VERSIONS_FILE=/versions.yaml \ + $(CALICO_BUILD) sh -c '\ + cd /go/src/$(PACKAGE_NAME) && \ + make startup' + +## Build allocate_ipip_addr.go +.PHONY: allocate-ipip-addr +allocate-ipip-addr: + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -i -o dist/allocate-ipip-addr $(LDFLAGS) allocateipip/allocate_ipip_addr.go + +dist/allocate-ipip-addr: $(ALLOCATE_IPIP_FILES) vendor + mkdir -p dist + mkdir -p .go-pkg-cache + docker run --rm \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + -v $(CURDIR)/.go-pkg-cache:/go/pkg/:rw \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):ro \ + -v $(VERSIONS_FILE):/versions.yaml:ro \ + -e VERSIONS_FILE=/versions.yaml \ + -v $(CURDIR)/dist:/go/src/$(PACKAGE_NAME)/dist \ + $(CALICO_BUILD) sh -c '\ + cd /go/src/$(PACKAGE_NAME) && \ + make allocate-ipip-addr' + +############################################################################### +# Tests +# - Support for running etcd (both securely and insecurely) +# - Running UTs and STs +############################################################################### +# These variables can be overridden by setting an environment variable. +############################################################################### +# Common build variables +# Path to the sources. +# Default value: directory with Makefile +SOURCE_DIR?=$(dir $(lastword $(MAKEFILE_LIST))) +SOURCE_DIR:=$(abspath $(SOURCE_DIR)) +LOCAL_IP_ENV?=$(shell ip route get 8.8.8.8 | head -1 | awk '{print $$7}') +ST_TO_RUN?=tests/st/ + +# Can exclude the slower tests with "-a '!slow'" +ST_OPTIONS?= +HOST_CHECKOUT_DIR?=$(shell pwd) + +# curl should failed on 404 +CURL=curl -sSf +## Generate the keys and certificates for running etcd with SSL. +ssl-certs: certs/.certificates.created ## Generate self-signed SSL certificates +certs/.certificates.created: + mkdir -p certs + $(CURL) -L "https://github.com/projectcalico/cfssl/releases/download/1.2.1/cfssl" -o certs/cfssl + $(CURL) -L "https://github.com/projectcalico/cfssl/releases/download/1.2.1/cfssljson" -o certs/cfssljson + chmod a+x certs/cfssl + chmod a+x certs/cfssljson + + certs/cfssl gencert -initca tests/st/ssl-config/ca-csr.json | certs/cfssljson -bare certs/ca + certs/cfssl gencert \ + -ca certs/ca.pem \ + -ca-key certs/ca-key.pem \ + -config tests/st/ssl-config/ca-config.json \ + tests/st/ssl-config/req-csr.json | certs/cfssljson -bare certs/client + certs/cfssl gencert \ + -ca certs/ca.pem \ + -ca-key certs/ca-key.pem \ + -config tests/st/ssl-config/ca-config.json \ + tests/st/ssl-config/req-csr.json | certs/cfssljson -bare certs/server + + touch certs/.certificates.created + +busybox.tar: + docker pull busybox:latest + docker save --output busybox.tar busybox:latest + +routereflector.tar: + docker pull calico/routereflector:latest + docker save --output routereflector.tar calico/routereflector:latest + +workload.tar: + cd workload && docker build -t workload . + docker save --output workload.tar workload + +stop-etcd: + @-docker rm -f calico-etcd calico-etcd-ssl + +.PHONY: run-etcd-ssl +## Run etcd in a container with SSL verification. Used primarily by STs. +run-etcd-ssl: certs/.certificates.created add-ssl-hostname + $(MAKE) stop-etcd + docker run --detach \ + --net=host \ + -v $(SOURCE_DIR)/certs:/etc/calico/certs \ + --name calico-etcd-ssl quay.io/coreos/etcd \ + etcd \ + --cert-file "/etc/calico/certs/server.pem" \ + --key-file "/etc/calico/certs/server-key.pem" \ + --trusted-ca-file "/etc/calico/certs/ca.pem" \ + --advertise-client-urls "https://etcd-authority-ssl:2379,https://localhost:2379" \ + --listen-client-urls "https://0.0.0.0:2379" + +IPT_ALLOW_ETCD:=-A INPUT -i docker0 -p tcp --dport 2379 -m comment --comment "calico-st-allow-etcd" -j ACCEPT + +.PHONY: st-checks +st-checks: + # Check that we're running as root. + test `id -u` -eq '0' || { echo "STs must be run as root to allow writes to /proc"; false; } + + # Insert an iptables rule to allow access from our test containers to etcd + # running on the host. + iptables-save | grep -q 'calico-st-allow-etcd' || iptables $(IPT_ALLOW_ETCD) + +## Run the STs in a container +.PHONY: st +st: dist/calicoctl dist/calicoctl-v1.0.2 busybox.tar routereflector.tar calico-node.tar workload.tar run-etcd-host calico_test.created + # Use the host, PID and network namespaces from the host. + # Privileged is needed since 'calico node' write to /proc (to enable ip_forwarding) + # Map the docker socket in so docker can be used from inside the container + # HOST_CHECKOUT_DIR is used for volume mounts on containers started by this one. + # All of code under test is mounted into the container. + # - This also provides access to calicoctl and the docker client + # $(MAKE) st-checks + docker run --uts=host \ + --pid=host \ + --net=host \ + --privileged \ + -e HOST_CHECKOUT_DIR=$(HOST_CHECKOUT_DIR) \ + -e DEBUG_FAILURES=$(DEBUG_FAILURES) \ + -e MY_IP=$(LOCAL_IP_ENV) \ + -e NODE_CONTAINER_NAME=$(NODE_CONTAINER_NAME) \ + --rm -ti \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(SOURCE_DIR):/code \ + $(SYSTEMTEST_CONTAINER) \ + sh -c 'nosetests $(ST_TO_RUN) -sv --nologcapture --with-xunit --xunit-file="/code/nosetests.xml" --with-timer $(ST_OPTIONS)' + $(MAKE) stop-etcd + +## Run the STs in a container using etcd with SSL certificate/key/CA verification. +.PHONY: st-ssl +st-ssl: run-etcd-ssl dist/calicoctl busybox.tar calico-node.tar routereflector.tar workload.tar calico_test.created + # Use the host, PID and network namespaces from the host. + # Privileged is needed since 'calico node' write to /proc (to enable ip_forwarding) + # Map the docker socket in so docker can be used from inside the container + # HOST_CHECKOUT_DIR is used for volume mounts on containers started by this one. + # All of code under test is mounted into the container. + # - This also provides access to calicoctl and the docker client + # Mount the full path to the etcd certs directory. + # - docker copies this directory directly from the host, but the + # calicoctl node command reads the files from the test container + $(MAKE) st-checks + docker run --uts=host \ + --pid=host \ + --net=host \ + --privileged \ + -e HOST_CHECKOUT_DIR=$(HOST_CHECKOUT_DIR) \ + -e DEBUG_FAILURES=$(DEBUG_FAILURES) \ + -e MY_IP=$(LOCAL_IP_ENV) \ + -e NODE_CONTAINER_NAME=$(NODE_CONTAINER_NAME) \ + -e ETCD_SCHEME=https \ + -e ETCD_CA_CERT_FILE=$(SOURCE_DIR)/certs/ca.pem \ + -e ETCD_CERT_FILE=$(SOURCE_DIR)/certs/client.pem \ + -e ETCD_KEY_FILE=$(SOURCE_DIR)/certs/client-key.pem \ + --rm -ti \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(SOURCE_DIR):/code \ + -v $(SOURCE_DIR)/certs:$(SOURCE_DIR)/certs \ + $(SYSTEMTEST_CONTAINER) \ + sh -c 'nosetests $(ST_TO_RUN) -sv --nologcapture --with-xunit --xunit-file="/code/nosetests.xml" --with-timer $(ST_OPTIONS)' + $(MAKE) stop-etcd + +.PHONY: add-ssl-hostname +add-ssl-hostname: + # Set "LOCAL_IP etcd-authority-ssl" in /etc/hosts to use as a hostname for etcd with ssl + if ! grep -q "etcd-authority-ssl" /etc/hosts; then \ + echo "\n# Host used by Calico's ETCD with SSL\n$(LOCAL_IP_ENV) etcd-authority-ssl" >> /etc/hosts; \ + fi + +## Etcd is used by the tests +.PHONY: run-etcd +run-etcd: + @-docker rm -f calico-etcd + docker run --detach \ + -p 2379:2379 \ + --name calico-etcd quay.io/coreos/etcd \ + etcd \ + --advertise-client-urls "http://$(LOCAL_IP_ENV):2379,http://127.0.0.1:2379" \ + --listen-client-urls "http://0.0.0.0:2379" + +## Etcd is used by the STs +.PHONY: run-etcd-host +run-etcd-host: + @-docker rm -f calico-etcd + docker run --detach \ + --net=host \ + --name calico-etcd quay.io/coreos/etcd \ + etcd \ + --advertise-client-urls "http://$(LOCAL_IP_ENV):2379,http://127.0.0.1:2379" \ + --listen-client-urls "http://0.0.0.0:2379" + +############################################################################### +# calico_node FVs +############################################################################### +.PHONY: node-fv +## Run the Functional Verification tests locally, must have local etcd running +node-fv: + # Run tests in random order find tests recursively (-r). + ginkgo -cover -r -skipPackage vendor startup allocateipip calicoclient + + @echo + @echo '+==============+' + @echo '| All coverage |' + @echo '+==============+' + @echo + @find . -iname '*.coverprofile' | xargs -I _ go tool cover -func=_ + + @echo + @echo '+==================+' + @echo '| Missing coverage |' + @echo '+==================+' + @echo + @find . -iname '*.coverprofile' | xargs -I _ go tool cover -func=_ | grep -v '100.0%' + +PHONY: node-test-containerized +## Run the tests in a container. Useful for CI, Mac dev. +node-test-containerized: vendor run-etcd-host + docker run --rm \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \ + -v $(VERSIONS_FILE):/versions.yaml:ro \ + -e VERSIONS_FILE=/versions.yaml \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + --net=host \ + $(CALICO_BUILD) sh -c 'cd /go/src/$(PACKAGE_NAME) && make node-fv' + + +# This depends on clean to ensure that dependent images get untagged and repulled +.PHONY: semaphore +semaphore: + # Clean up unwanted files to free disk space. + bash -c 'rm -rf /usr/local/golang /opt /var/lib/mongodb /usr/lib/jvm /home/runner/{.npm,.phpbrew,.phpunit,.kerl,.kiex,.lein,.nvm,.npm,.phpbrew,.rbenv}' + + # Run the containerized UTs first. + $(MAKE) node-test-containerized + + # Actually run the tests (refreshing the images as required), we only run a + # small subset of the tests for testing SSL support. These tests are run + # using "latest" tagged images. + $(MAKE) $(NODE_CONTAINER_NAME) st + ST_TO_RUN=tests/st/policy $(MAKE) st-ssl + + # Assumes that a few environment variables exist - BRANCH_NAME PULL_REQUEST_NUMBER + # If this isn't a PR, then push :BRANCHNAME tagged and :CALICO_VER + # tagged images to Dockerhub and quay for calico/node + set -e; \ + if [ -z $$PULL_REQUEST_NUMBER ]; then \ + docker tag $(NODE_CONTAINER_NAME) quay.io/$(NODE_CONTAINER_NAME):$$BRANCH_NAME && \ + docker push quay.io/$(NODE_CONTAINER_NAME):$$BRANCH_NAME; \ + \ + docker tag $(NODE_CONTAINER_NAME) $(NODE_CONTAINER_NAME):$$BRANCH_NAME && \ + docker push $(NODE_CONTAINER_NAME):$$BRANCH_NAME; \ + \ + docker tag $(NODE_CONTAINER_NAME) quay.io/$(NODE_CONTAINER_NAME):$(CALICO_VER) && \ + docker push quay.io/$(NODE_CONTAINER_NAME):$(CALICO_VER); \ + \ + docker tag $(NODE_CONTAINER_NAME) $(NODE_CONTAINER_NAME):$(CALICO_VER) && \ + docker push $(NODE_CONTAINER_NAME):$(CALICO_VER); \ + fi + +release: clean + git tag $(CALICO_VER) + # Build the calico/node images. + $(MAKE) $(NODE_CONTAINER_NAME) + + # Retag images with corect version and quay + docker tag $(NODE_CONTAINER_NAME) $(NODE_CONTAINER_NAME):$(CALICO_VER) + docker tag $(NODE_CONTAINER_NAME) quay.io/$(NODE_CONTAINER_NAME):$(CALICO_VER) + docker tag $(NODE_CONTAINER_NAME) quay.io/$(NODE_CONTAINER_NAME):latest + + # Check that images were created recently and that the IDs of the versioned and latest images match + @docker images --format "{{.CreatedAt}}\tID:{{.ID}}\t{{.Repository}}:{{.Tag}}" $(NODE_CONTAINER_NAME) + @docker images --format "{{.CreatedAt}}\tID:{{.ID}}\t{{.Repository}}:{{.Tag}}" $(NODE_CONTAINER_NAME):$(CALICO_VER) + + # Check that the images container the right sub-components + docker run $(NODE_CONTAINER_NAME) calico-felix --version + docker run $(NODE_CONTAINER_NAME) libnetwork-plugin -v + + @echo "Now push the tag and images." + @echo "git push origin $(CALICO_VER)" + + @echo "docker push $(NODE_CONTAINER_NAME):$(CALICO_VER)" + @echo "docker push quay.io/$(NODE_CONTAINER_NAME):$(CALICO_VER)" + @echo "docker push $(NODE_CONTAINER_NAME):latest" + @echo "docker push quay.io/$(NODE_CONTAINER_NAME):latest" + @echo "See RELEASING.md for detailed instructions." + +## Clean enough that a new release build will be clean +clean: + find . -name '*.created' -exec rm -f {} + + find . -name '*.pyc' -exec rm -f {} + + rm -rf dist build certs *.tar vendor $(NODE_CONTAINER_DIR)/filesystem/bin + + # Delete images that we built in this repo + docker rmi $(NODE_CONTAINER_NAME):latest || true + docker rmi $(SYSTEMTEST_CONTAINER) || true + + # Retag and remove external images so that they will be pulled again + # We avoid just deleting the image. We didn't build them here so it would be impolite to delete it. + docker tag $(FELIX_CONTAINER_NAME) $(FELIX_CONTAINER_NAME)-backup && docker rmi $(FELIX_CONTAINER_NAME) || true + +.PHONY: help +## Display this help text +help: # Some kind of magic from https://gist.github.com/rcmachado/af3db315e31383502660 + $(info Available targets) + @awk '/^[a-zA-Z\-\_0-9\/]+:/ { \ + nb = sub( /^## /, "", helpMsg ); \ + if(nb == 0) { \ + helpMsg = $$0; \ + nb = sub( /^[^:]*:.* ## /, "", helpMsg ); \ + } \ + if (nb) \ + printf "\033[1;31m%-" width "s\033[0m %s\n", $$1, helpMsg; \ + } \ + { helpMsg = $$0 }' \ + width=20 \ + $(MAKEFILE_LIST) + diff --git a/allocateipip/allocate_ipip_addr.go b/allocateipip/allocate_ipip_addr.go new file mode 100644 index 000000000..9a5e294d5 --- /dev/null +++ b/allocateipip/allocate_ipip_addr.go @@ -0,0 +1,145 @@ +package main + +import ( + "os" + + log "github.com/Sirupsen/logrus" + + "github.com/projectcalico/calico/calico_node/calicoclient" + "github.com/projectcalico/libcalico-go/lib/api" + "github.com/projectcalico/libcalico-go/lib/client" + "github.com/projectcalico/libcalico-go/lib/net" +) + +// This file contains the main processing for the allocate_ipip_addr binary +// used by calico/node to set the host's tunnel address to an IPIP-enabled +// address if there are any available, otherwise it removes any tunnel address +// that is configured. + +func main() { + // Load the client config from environment. + cfg, c := calicoclient.CreateClient() + + // This is a no-op for KDD. + if cfg.Spec.DatastoreType == api.Kubernetes { + log.Info("Kubernetes datastore driver handles IPIP allocation - no op") + return + } + + // The allocate_ipip_addr binary is only ever invoked _after_ the + // startup binary has been invoked and the modified environments have + // been sourced. Therefore, the NODENAME environment will always be + // set at this point. + nodename := os.Getenv("NODENAME") + if nodename == "" { + log.Panic("NODENAME environment is not set") + } + + // Query the IPIP enabled pools and either configure the tunnel + // address, or remove it. + if cidrs := getIPIPEnabledPoolCIDRs(c); len(cidrs) > 0 { + ensureHostTunnelAddress(c, nodename, cidrs) + } else { + removeHostTunnelAddr(c, nodename) + } +} + +// ensureHostTunnelAddress that ensures the host has a valid IP address for the +// IPIP tunnel device. This must be an IP address claimed from one of the IPIP +// pools. This function handles re-allocating the address if it finds an +// existing address that is not from an IPIP pool. +func ensureHostTunnelAddress(c *client.Client, nodename string, ipipCidrs []net.IPNet) { + log.WithField("Node", nodename).Debug("Ensure IPIP tunnel address is set") + + // Get the currently configured IPIP address. + if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil { + log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address") + } else if ipAddr == nil { + // The IPIP tunnel has no IP address assigned, assign one. + log.Debug("IPIP tunnel is not assigned - assign IP") + assignHostTunnelAddr(c, nodename, ipipCidrs) + } else if isIpInPool(ipAddr, ipipCidrs) { + // The IPIP tunnel address is still valid, so leave as it. + log.WithField("IP", ipAddr.String()).Info("IPIP tunnel address is still valid") + } else { + // The address that is currently assigned is no longer part + // of an IPIP pool, so release the IP, and reassign. + log.WithField("IP", ipAddr.String()).Info("Reassigning IPIP tunnel address") + + ipsToRelease := []net.IP{*ipAddr} + _, err := c.IPAM().ReleaseIPs(ipsToRelease) + if err != nil { + log.WithField("IP", ipAddr.String()).WithError(err).Fatal("Error releasing non IPIP address") + } + + // Assign a new tunnel address. + assignHostTunnelAddr(c, nodename, ipipCidrs) + } +} + +// removeHostTunnelAddr removes any existing IP address for this host's IPIP +// tunnel device and releases the IP from IPAM. If no IP is assigned this function +// is a no-op. +func removeHostTunnelAddr(c *client.Client, nodename string) { + if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil { + log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address for cleanup") + } else if ipAddr == nil { + log.Debug("No IPIP tunnel address assigned, and not required") + } else if _, err := c.IPAM().ReleaseIPs([]net.IP{*ipAddr}); err != nil { + log.WithError(err).WithField("IP", ipAddr.String()).Fatal("Error releasing IPIP address from IPAM") + } else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, nil); err != nil { + log.WithError(err).Fatal("Unable to remove IPIP tunnel address") + } +} + +// assignHostTunnelAddr claims an IPIP-enabled IP address from the first pool +// with some space. Stores the result in the host's config as its tunnel +// address. +func assignHostTunnelAddr(c *client.Client, nodename string, ipipCidrs []net.IPNet) { + args := client.AutoAssignArgs{ + Num4: 1, + Num6: 0, + HandleID: nil, + Attrs: nil, + Hostname: nodename, + IPv4Pools: ipipCidrs, + } + + if ipv4Addrs, _, err := c.IPAM().AutoAssign(args); err != nil { + log.WithError(err).Fatal("Unable to autoassign an address for IPIP") + } else if len(ipv4Addrs) == 0 { + log.Fatal("Unable to autoassign an address for IPIP - pools are likely exhausted.") + } else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, &ipv4Addrs[0]); err != nil { + log.WithError(err).WithField("IP", ipv4Addrs[0].String()).Fatal("Unable to set IPIP tunnel address") + } else { + log.WithField("IP", ipv4Addrs[0].String()).Info("Set IPIP tunnel address") + } +} + +// isIpInPool returns if the IP address is in one of the supplied pools. +func isIpInPool(ipAddr *net.IP, ipipCidrs []net.IPNet) bool { + for _, cidr := range ipipCidrs { + if cidr.Contains(ipAddr.IP) { + return true + } + } + return false +} + +// getIPIPEnabledPools returns all IPIP enabled pools. +func getIPIPEnabledPoolCIDRs(c *client.Client) []net.IPNet { + meta := api.IPPoolMetadata{} + ipPoolList, err := c.IPPools().List(meta) + if err != nil { + log.WithError(err).Fatal("Unable to query IP pool configuration") + } + + var cidrs []net.IPNet + for _, ipPool := range ipPoolList.Items { + // Check if IPIP is enabled in the IP pool, the IP pool is not disabled, and it is IPv4 pool since we don't support IPIP with IPv6. + if ipPool.Spec.IPIP != nil && ipPool.Spec.IPIP.Enabled && !ipPool.Spec.Disabled && ipPool.Metadata.CIDR.Version() == 4 { + cidrs = append(cidrs, ipPool.Metadata.CIDR) + } + } + return cidrs +} diff --git a/allocateipip/allocateipip_suite_test.go b/allocateipip/allocateipip_suite_test.go new file mode 100644 index 000000000..66bbb9e89 --- /dev/null +++ b/allocateipip/allocateipip_suite_test.go @@ -0,0 +1,19 @@ +package main_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" + + "github.com/projectcalico/libcalico-go/lib/testutils" +) + +func init() { + testutils.HookLogrusForGinkgo() +} + +func TestCommands(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AllocateIPIPAddr Suite") +} diff --git a/calico_test/Dockerfile.calico_test b/calico_test/Dockerfile.calico_test new file mode 100644 index 000000000..55d5c1296 --- /dev/null +++ b/calico_test/Dockerfile.calico_test @@ -0,0 +1,55 @@ +# Copyright (c) 2015 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +### calico/test +# This image is used by various calico repositories and components to run UTs +# and STs. It has libcalico, nose, and other common python libraries +# already installed +# +# For UTs: +# - volume mount in python code that uses libcalico +# - volume mount in your unit tests for this code +# - run 'nosetests' +# +# This container can also be used for running STs written in python. This +# eliminates all dependencies besides docker on the host system to enable +# running of the ST frameworks. +# To run: +# - volume mount the docker socket, allowing the STs to launch docker +# containers alongside itself. +# - eliminate most isolation, (--uts=host --pid=host --net=host --privileged) +# - volume mount your ST source code +# - run 'nosetests' +FROM docker:1.13.0 +MAINTAINER Tom Denham + +# Running STs in this container requires that it has all dependencies installed +# for executing the tests. Install these dependencies: +RUN apk add --update python python-dev py2-pip py-setuptools openssl-dev libffi-dev tshark \ + netcat-openbsd iptables ip6tables iproute2 iputils ipset curl && \ + echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ + rm -rf /var/cache/apk/* + +COPY requirements.txt /requirements.txt +RUN pip install -r /requirements.txt + +# Install etcdctl +RUN wget https://github.com/coreos/etcd/releases/download/v2.3.3/etcd-v2.3.3-linux-amd64.tar.gz && \ + tar -xzf etcd-v2.3.3-linux-amd64.tar.gz && \ + cd etcd-v2.3.3-linux-amd64 && \ + ln -s etcdctl /usr/local/bin/ + +# The container is used by mounting the code-under-test to /code +WORKDIR /code/ diff --git a/calico_test/requirements.txt b/calico_test/requirements.txt new file mode 100644 index 000000000..3cd543ca6 --- /dev/null +++ b/calico_test/requirements.txt @@ -0,0 +1,7 @@ +nose +nose-timer +nose-parameterized +netaddr +pyyaml +simplejson +deepdiff diff --git a/calicoclient/calicoclient.go b/calicoclient/calicoclient.go new file mode 100644 index 000000000..cc142173b --- /dev/null +++ b/calicoclient/calicoclient.go @@ -0,0 +1,27 @@ +package calicoclient + +import ( + "fmt" + "os" + + "github.com/projectcalico/libcalico-go/lib/api" + "github.com/projectcalico/libcalico-go/lib/client" +) + +// CreateClient loads the client config from environments and creates the +// Calico client. +func CreateClient() (*api.CalicoAPIConfig, *client.Client) { + // Load the client config from environment. + cfg, err := client.LoadClientConfig("") + if err != nil { + fmt.Printf("ERROR: Error loading datastore config: %s", err) + os.Exit(1) + } + c, err := client.New(*cfg) + if err != nil { + fmt.Printf("ERROR: Error accessing the Calico datastore: %s", err) + os.Exit(1) + } + + return cfg, c +} diff --git a/calicoclient/calicoclient_suite_test.go b/calicoclient/calicoclient_suite_test.go new file mode 100644 index 000000000..bce3d2fb6 --- /dev/null +++ b/calicoclient/calicoclient_suite_test.go @@ -0,0 +1,19 @@ +package calicoclient_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" + + "github.com/projectcalico/libcalico-go/lib/testutils" +) + +func init() { + testutils.HookLogrusForGinkgo() +} + +func TestCommands(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Calicoclient Suite") +} diff --git a/filesystem/etc/calico/confd/conf.d/bird.toml.toml b/filesystem/etc/calico/confd/conf.d/bird.toml.toml new file mode 100644 index 000000000..51e32649a --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/bird.toml.toml @@ -0,0 +1,8 @@ +[template] +src = "bird.toml.template" +dest = "/etc/calico/confd/conf.d/bird.toml" +prefix = "/calico/bgp/v1/global" +keys = [ + "/node_mesh", +] +reload_cmd = "restart-calico-confd" diff --git a/filesystem/etc/calico/confd/conf.d/bird6.toml.toml b/filesystem/etc/calico/confd/conf.d/bird6.toml.toml new file mode 100644 index 000000000..c9cef2215 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/bird6.toml.toml @@ -0,0 +1,8 @@ +[template] +src = "bird6.toml.template" +dest = "/etc/calico/confd/conf.d/bird6.toml" +prefix = "/calico/bgp/v1/global" +keys = [ + "/node_mesh" +] +reload_cmd = "restart-calico-confd" diff --git a/filesystem/etc/calico/confd/conf.d/bird6_ipam.toml b/filesystem/etc/calico/confd/conf.d/bird6_ipam.toml new file mode 100644 index 000000000..6d90d3410 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/bird6_ipam.toml @@ -0,0 +1,8 @@ +[template] +src = "bird6_ipam.cfg.template" +dest = "/etc/calico/confd/config/bird6_ipam.cfg" +prefix = "/calico/v1/ipam/v6" +keys = [ + "/pool", +] +reload_cmd = "pkill -HUP bird6 || true" diff --git a/filesystem/etc/calico/confd/conf.d/bird_ipam.toml b/filesystem/etc/calico/confd/conf.d/bird_ipam.toml new file mode 100644 index 000000000..30938a7b7 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/bird_ipam.toml @@ -0,0 +1,8 @@ +[template] +src = "bird_ipam.cfg.template" +dest = "/etc/calico/confd/config/bird_ipam.cfg" +prefix = "/calico/v1/ipam/v4" +keys = [ + "/pool", +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/conf.d/custom_filters.toml b/filesystem/etc/calico/confd/conf.d/custom_filters.toml new file mode 100644 index 000000000..dc7a5eeb3 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/custom_filters.toml @@ -0,0 +1,8 @@ +[template] +src = "custom_filters.cfg.template" +dest = "/etc/calico/confd/config/custom_filters.cfg" +prefix = "/calico/bgp/v1/global/custom_filters" +keys = [ + "/v4", +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/conf.d/custom_filters6.toml b/filesystem/etc/calico/confd/conf.d/custom_filters6.toml new file mode 100644 index 000000000..47afdc835 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/custom_filters6.toml @@ -0,0 +1,8 @@ +[template] +src = "custom_filters6.cfg.template" +dest = "/etc/calico/confd/config/custom_filters6.cfg" +prefix = "/calico/bgp/v1/global/custom_filters" +keys = [ + "/v6", +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/conf.d/tunl-ip.toml b/filesystem/etc/calico/confd/conf.d/tunl-ip.toml new file mode 100644 index 000000000..ef3433d83 --- /dev/null +++ b/filesystem/etc/calico/confd/conf.d/tunl-ip.toml @@ -0,0 +1,8 @@ +[template] +src = "tunl-ip.template" +dest = "/tmp/tunl-ip" +prefix = "/calico/v1/ipam/v4" +keys = [ + "/pool", +] +reload_cmd = "allocate-ipip-addr" diff --git a/filesystem/etc/calico/confd/config/.gitkeep b/filesystem/etc/calico/confd/config/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/filesystem/etc/calico/confd/templates/README.md b/filesystem/etc/calico/confd/templates/README.md new file mode 100644 index 000000000..28da444b4 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/README.md @@ -0,0 +1,47 @@ +The following is a summary of the templates defined in this directory: + +### bird.toml.template / bird6.toml.tempate + +Referenced by bird.toml.toml and bird6.toml.toml. + +These templates write out a TOML file (bird.toml and bird6.toml) that is used +to tell confd which set of the main BIRD templates to use. + +Based off the node_mesh parameter, the TOML file generated either points to the +full-mesh config files, or the no-mesh config files. + +Once confd writes out the appropriate TOML file, confd is restarted to pick up +the change to the bird.toml and bird6.toml files. Since there are two sets +of changes, a change to the node_mesh parameter will result in a double +restart of confd. + +In short, this is used by confd to generate its own configuration. + + +### bird_ipam.cfg.template / bird6_ipam.cfg.template + +Referenced by bird_ipam.toml and bird6_ipam.toml. + +These templates write out the route filters based on IPAM configuration. This +is inherited by the main BIRD configuration file. + +It is separated out from the main BIRD configuration file because it watches a +different sub-tree in etcd. This allows confd to watch a smaller portion of +the tree to reduce churn. + + +### bird.cfg.mesh.template / bird6.cfg.mesh.template + +Referenced by the confd-generated bird.toml and bird6.toml files. + +These templates write out the main BIRD configuration when the full +node-to-node mesh is enabled. + + +### bird.cfg.no-mesh.template / bird6.cfg.no-mesh.template + +Referenced by the confd-generated bird.toml and bird6.toml files. + +These templates write out the main BIRD configuration when the full +node-to-node mesh is disabled. +[![Analytics](https://calico-ga-beacon.appspot.com/UA-52125893-3/calicoctl/calico_node/filesystem/templates/README.md?pixel)](https://github.com/igrigorik/ga-beacon) diff --git a/filesystem/etc/calico/confd/templates/bird.cfg.mesh.template b/filesystem/etc/calico/confd/templates/bird.cfg.mesh.template new file mode 100644 index 000000000..66e58301a --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird.cfg.mesh.template @@ -0,0 +1,99 @@ +# Generated by confd +include "bird_aggr.cfg"; +include "custom_filters.cfg"; +include "bird_ipam.cfg"; +{{$node_ip_key := printf "/host/%s/ip_addr_v4" (getenv "NODENAME")}}{{$node_ip := getv $node_ip_key}} + +router id {{$node_ip}}; + +{{define "LOGGING"}} +{{$node_logging_key := printf "/host/%s/loglevel" (getenv "NODENAME")}}{{if exists $node_logging_key}}{{$logging := getv $node_logging_key}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else if exists "/global/loglevel"}}{{$logging := getv "/global/loglevel"}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else}} debug { states };{{end}} +{{end}} + +# Configure synchronization between routing tables and kernel. +protocol kernel { + learn; # Learn all alien routes from the kernel + persist; # Don't remove routes on bird shutdown + scan time 2; # Scan kernel routing table every 2 seconds + import all; + export filter calico_ipip; # Default is export none + graceful restart; # Turn on graceful restart to reduce potential flaps in + # routes when reloading BIRD configuration. With a full + # automatic mesh, there is no way to prevent BGP from + # flapping since multiple nodes update their BGP + # configuration at the same time, GR is not guaranteed to + # work correctly in this scenario. +} + +# Watch interface up/down events. +protocol device { + {{template "LOGGING"}} + scan time 2; # Scan interfaces every 2 seconds +} + +protocol direct { + {{template "LOGGING"}} + interface -"cali*", "*"; # Exclude cali* but include everything else. +} + +{{$node_as_key := printf "/host/%s/as_num" (getenv "NODENAME")}} +# Template for all BGP clients +template bgp bgp_template { + {{template "LOGGING"}} + description "Connection to BGP peer"; + local as {{if exists $node_as_key}}{{getv $node_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + multihop; + gateway recursive; # This should be the default, but just in case. + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export filter calico_pools; # Only want to export routes for workloads. + next hop self; # Disable next hop processing and always advertise our + # local address as nexthop + source address {{$node_ip}}; # The local address we use for the TCP connection + add paths on; + graceful restart; # See comment in kernel section about graceful restart. +} + +# ------------- Node-to-node mesh ------------- +{{if (json (getv "/global/node_mesh")).enabled}} +{{range $host := lsdir "/host"}} +{{$onode_as_key := printf "/host/%s/as_num" .}} +{{$onode_ip_key := printf "/host/%s/ip_addr_v4" .}}{{if exists $onode_ip_key}}{{$onode_ip := getv $onode_ip_key}} +{{$nums := split $onode_ip "."}}{{$id := join $nums "_"}} +# For peer {{$onode_ip_key}} +{{if eq $onode_ip ($node_ip) }}# Skipping ourselves ({{$node_ip}}) +{{else if ne "" $onode_ip}}protocol bgp Mesh_{{$id}} from bgp_template { + neighbor {{$onode_ip}} as {{if exists $onode_as_key}}{{getv $onode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; +}{{end}}{{end}}{{end}} +{{else}} +# Node-to-node mesh disabled +{{end}} + + +# ------------- Global peers ------------- +{{if ls "/global/peer_v4"}} +{{range gets "/global/peer_v4/*"}}{{$data := json .Value}} +{{$nums := split $data.ip "."}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Global_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No global peers configured.{{end}} + + +# ------------- Node-specific peers ------------- +{{$node_peers_key := printf "/host/%s/peer_v4" (getenv "NODENAME")}} +{{if ls $node_peers_key}} +{{range gets (printf "%s/*" $node_peers_key)}}{{$data := json .Value}} +{{$nums := split $data.ip "."}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Node_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No node-specific peers configured.{{end}} diff --git a/filesystem/etc/calico/confd/templates/bird.cfg.no-mesh.template b/filesystem/etc/calico/confd/templates/bird.cfg.no-mesh.template new file mode 100644 index 000000000..1c81bf902 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird.cfg.no-mesh.template @@ -0,0 +1,84 @@ +# Generated by confd +include "bird_aggr.cfg"; +include "custom_filters.cfg"; +include "bird_ipam.cfg"; +{{$node_ip_key := printf "/host/%s/ip_addr_v4" (getenv "NODENAME")}}{{$node_ip := getv $node_ip_key}} + +router id {{$node_ip}}; + +{{define "LOGGING"}} +{{$node_logging_key := printf "/host/%s/loglevel" (getenv "NODENAME")}}{{if exists $node_logging_key}}{{$logging := getv $node_logging_key}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else if exists "/global/loglevel"}}{{$logging := getv "/global/loglevel"}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else}} debug { states };{{end}} +{{end}} + +# Configure synchronization between routing tables and kernel. +protocol kernel { + learn; # Learn all alien routes from the kernel + persist; # Don't remove routes on bird shutdown + scan time 2; # Scan kernel routing table every 2 seconds + import all; + export filter calico_ipip; # Default is export none + graceful restart; # Turn on graceful restart to reduce potential flaps in + # routes when reloading BIRD configuration. With a full + # automatic mesh, there is no way to prevent BGP from + # flapping since multiple nodes update their BGP + # configuration at the same time, GR is not guaranteed to + # work correctly in this scenario. +} + +# Watch interface up/down events. +protocol device { + {{template "LOGGING"}} + scan time 2; # Scan interfaces every 2 seconds +} + +protocol direct { + {{template "LOGGING"}} + interface -"cali*", "*"; # Exclude cali* but include everything else. +} + +{{$node_as_key := printf "/host/%s/as_num" (getenv "NODENAME")}} +# Template for all BGP clients +template bgp bgp_template { + {{template "LOGGING"}} + description "Connection to BGP peer"; + local as {{if exists $node_as_key}}{{getv $node_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + multihop; + gateway recursive; # This should be the default, but just in case. + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export filter calico_pools; # Only want to export routes for workloads. + next hop self; # Disable next hop processing and always advertise our + # local address as nexthop + source address {{$node_ip}}; # The local address we use for the TCP connection + add paths on; + graceful restart; # See comment in kernel section about graceful restart. +} + + +# ------------- Global peers ------------- +{{if ls "/global/peer_v4"}} +{{range gets "/global/peer_v4/*"}}{{$data := json .Value}} +{{$nums := split $data.ip "."}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Global_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No global peers configured.{{end}} + + +# ------------- Node-specific peers ------------- +{{$node_peers_key := printf "/host/%s/peer_v4" (getenv "NODENAME")}} +{{if ls $node_peers_key}} +{{range gets (printf "%s/*" $node_peers_key)}}{{$data := json .Value}} +{{$nums := split $data.ip "."}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Node_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No node-specific peers configured.{{end}} diff --git a/filesystem/etc/calico/confd/templates/bird.toml.template b/filesystem/etc/calico/confd/templates/bird.toml.template new file mode 100644 index 000000000..0fbde9ae3 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird.toml.template @@ -0,0 +1,10 @@ +[template] +src = "bird.cfg.{{if (json (getv "/node_mesh")).enabled}}mesh{{else}}no-mesh{{end}}.template" +dest = "/etc/calico/confd/config/bird.cfg" +prefix = "/calico/bgp/v1" +keys = [ + {{if (json (getv "/node_mesh")).enabled}}"/host"{{else}}"/host/{{getenv "NODENAME"}}"{{end}}, + "/global" +] +check_cmd = "bird -p -c {{"{{"}}.src{{"}}"}}" +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/templates/bird6.cfg.mesh.template b/filesystem/etc/calico/confd/templates/bird6.cfg.mesh.template new file mode 100644 index 000000000..694f9dfd7 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird6.cfg.mesh.template @@ -0,0 +1,103 @@ +# Generated by confd +include "bird6_aggr.cfg"; +include "custom_filters6.cfg"; +include "bird6_ipam.cfg"; +{{$node_ip_key := printf "/host/%s/ip_addr_v4" (getenv "NODENAME")}}{{$node_ip := getv $node_ip_key}} +{{$node_ip6_key := printf "/host/%s/ip_addr_v6" (getenv "NODENAME")}}{{$node_ip6 := getv $node_ip6_key}} + +router id {{$node_ip}}; # Use IPv4 address since router id is 4 octets, even in MP-BGP + +{{define "LOGGING"}} +{{$node_logging_key := printf "/host/%s/loglevel" (getenv "NODENAME")}}{{if exists $node_logging_key}}{{$logging := getv $node_logging_key}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else if exists "/global/loglevel"}}{{$logging := getv "/global/loglevel"}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else}} debug { states };{{end}} +{{end}} + +# Configure synchronization between routing tables and kernel. +protocol kernel { + learn; # Learn all alien routes from the kernel + persist; # Don't remove routes on bird shutdown + scan time 2; # Scan kernel routing table every 2 seconds + import all; + export all; # Default is export none + graceful restart; # Turn on graceful restart to reduce potential flaps in + # routes when reloading BIRD configuration. With a full + # automatic mesh, there is no way to prevent BGP from + # flapping since multiple nodes update their BGP + # configuration at the same time, GR is not guaranteed to + # work correctly in this scenario. +} + +# Watch interface up/down events. +protocol device { + {{template "LOGGING"}} + scan time 2; # Scan interfaces every 2 seconds +} + +protocol direct { + {{template "LOGGING"}} + interface -"cali*", "*"; # Exclude cali* but include everything else. +} + +{{if eq "" ($node_ip6)}}# IPv6 disabled on this node. +{{else}}{{$node_as_key := printf "/host/%s/as_num" (getenv "NODENAME")}} +# Template for all BGP clients +template bgp bgp_template { + {{template "LOGGING"}} + description "Connection to BGP peer"; + local as {{if exists $node_as_key}}{{getv $node_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + multihop; + gateway recursive; # This should be the default, but just in case. + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export filter calico_pools; # Only want to export routes for workloads. + next hop self; # Disable next hop processing and always advertise our + # local address as nexthop + source address {{$node_ip6}}; # The local address we use for the TCP connection + add paths on; + graceful restart; # See comment in kernel section about graceful restart. +} + +# ------------- Node-to-node mesh ------------- +{{if (json (getv "/global/node_mesh")).enabled}} +{{range $host := lsdir "/host"}} +{{$onode_as_key := printf "/host/%s/as_num" .}} +{{$onode_ip_key := printf "/host/%s/ip_addr_v6" .}}{{if exists $onode_ip_key}}{{$onode_ip := getv $onode_ip_key}} +{{$nums := split $onode_ip ":"}}{{$id := join $nums "_"}} +# For peer {{$onode_ip_key}} +{{if eq $onode_ip ($node_ip6) }}# Skipping ourselves ({{$node_ip6}}) +{{else if eq "" $onode_ip}}# No IPv6 address configured for this node +{{else}}protocol bgp Mesh_{{$id}} from bgp_template { + neighbor {{$onode_ip}} as {{if exists $onode_as_key}}{{getv $onode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; +}{{end}}{{end}}{{end}} +{{else}} +# Node-to-node mesh disabled +{{end}} + + +# ------------- Global peers ------------- +{{if ls "/global/peer_v6"}} +{{range gets "/global/peer_v6/*"}}{{$data := json .Value}} +{{$nums := split $data.ip ":"}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Global_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No global peers configured.{{end}} + + +# ------------- Node-specific peers ------------- +{{$node_peers_key := printf "/host/%s/peer_v6" (getenv "NODENAME")}} +{{if ls $node_peers_key}} +{{range gets (printf "%s/*" $node_peers_key)}}{{$data := json .Value}} +{{$nums := split $data.ip ":"}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Node_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No node-specific peers configured.{{end}} +{{end}} diff --git a/filesystem/etc/calico/confd/templates/bird6.cfg.no-mesh.template b/filesystem/etc/calico/confd/templates/bird6.cfg.no-mesh.template new file mode 100644 index 000000000..5206b5422 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird6.cfg.no-mesh.template @@ -0,0 +1,87 @@ +# Generated by confd +include "bird6_aggr.cfg"; +include "custom_filters6.cfg"; +include "bird6_ipam.cfg"; +{{$node_ip_key := printf "/host/%s/ip_addr_v4" (getenv "NODENAME")}}{{$node_ip := getv $node_ip_key}} +{{$node_ip6_key := printf "/host/%s/ip_addr_v6" (getenv "NODENAME")}}{{$node_ip6 := getv $node_ip6_key}} + +router id {{$node_ip}}; # Use IPv4 address since router id is 4 octets, even in MP-BGP + +{{define "LOGGING"}} +{{$node_logging_key := printf "/host/%s/loglevel" (getenv "NODENAME")}}{{if exists $node_logging_key}}{{$logging := getv $node_logging_key}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else if exists "/global/loglevel"}}{{$logging := getv "/global/loglevel"}} +{{if eq $logging "debug"}} debug all;{{else if ne $logging "none"}} debug { states };{{end}} +{{else}} debug { states };{{end}} +{{end}} + +# Configure synchronization between routing tables and kernel. +protocol kernel { + learn; # Learn all alien routes from the kernel + persist; # Don't remove routes on bird shutdown + scan time 2; # Scan kernel routing table every 2 seconds + import all; + export all; # Default is export none + graceful restart; # Turn on graceful restart to reduce potential flaps in + # routes when reloading BIRD configuration. With a full + # automatic mesh, there is no way to prevent BGP from + # flapping since multiple nodes update their BGP + # configuration at the same time, GR is not guaranteed to + # work correctly in this scenario. +} + +# Watch interface up/down events. +protocol device { + {{template "LOGGING"}} + scan time 2; # Scan interfaces every 2 seconds +} + +protocol direct { + {{template "LOGGING"}} + interface -"cali*", "*"; # Exclude cali* but include everything else. +} + +{{if eq "" ($node_ip6)}}# IPv6 disabled on this node. +{{else}}{{$node_as_key := printf "/host/%s/as_num" (getenv "NODENAME")}} +# Template for all BGP clients +template bgp bgp_template { + {{template "LOGGING"}} + description "Connection to BGP peer"; + local as {{if exists $node_as_key}}{{getv $node_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + multihop; + gateway recursive; # This should be the default, but just in case. + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export filter calico_pools; # Only want to export routes for workloads. + next hop self; # Disable next hop processing and always advertise our + # local address as nexthop + source address {{$node_ip6}}; # The local address we use for the TCP connection + add paths on; + graceful restart; # See comment in kernel section about graceful restart. +} + + +# ------------- Global peers ------------- +{{if ls "/global/peer_v6"}} +{{range gets "/global/peer_v6/*"}}{{$data := json .Value}} +{{$nums := split $data.ip ":"}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Global_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No global peers configured.{{end}} + + +# ------------- Node-specific peers ------------- +{{$node_peers_key := printf "/host/%s/peer_v6" (getenv "NODENAME")}} +{{if ls $node_peers_key}} +{{range gets (printf "%s/*" $node_peers_key)}}{{$data := json .Value}} +{{$nums := split $data.ip ":"}}{{$id := join $nums "_"}} +# For peer {{.Key}} +protocol bgp Node_{{$id}} from bgp_template { + neighbor {{$data.ip}} as {{$data.as_num}}; +} +{{end}} +{{else}}# No node-specific peers configured.{{end}} +{{end}} diff --git a/filesystem/etc/calico/confd/templates/bird6.toml.template b/filesystem/etc/calico/confd/templates/bird6.toml.template new file mode 100644 index 000000000..bfaf2a4e9 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird6.toml.template @@ -0,0 +1,10 @@ +[template] +src = "bird6.cfg.{{if (json (getv "/node_mesh")).enabled}}mesh{{else}}no-mesh{{end}}.template" +dest = "/etc/calico/confd/config/bird6.cfg" +prefix = "/calico/bgp/v1" +keys = [ + {{if (json (getv "/node_mesh")).enabled}}"/host"{{else}}"/host/{{getenv "NODENAME"}}"{{end}}, + "/global" +] +check_cmd = "bird6 -p -c {{"{{"}}.src{{"}}"}}" +reload_cmd = "pkill -HUP bird6 || true" diff --git a/filesystem/etc/calico/confd/templates/bird6_aggr.toml.template b/filesystem/etc/calico/confd/templates/bird6_aggr.toml.template new file mode 100644 index 000000000..494816e50 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird6_aggr.toml.template @@ -0,0 +1,8 @@ +[template] +src = "bird_aggr.cfg.template" +dest = "/etc/calico/confd/config/bird6_aggr.cfg" +prefix = "/calico/ipam/v2/host/NODENAME/ipv6/block" +keys = [ + "/", +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/templates/bird6_ipam.cfg.template b/filesystem/etc/calico/confd/templates/bird6_ipam.cfg.template new file mode 100644 index 000000000..601ecc3ae --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird6_ipam.cfg.template @@ -0,0 +1,11 @@ +# Generated by confd +filter calico_pools { + calico_aggr(); + custom_filters(); +{{range ls "/pool"}}{{$data := json (getv (printf "/pool/%s" .))}} + if ( net ~ {{$data.cidr}} ) then { + accept; + } +{{end}} + reject; +} diff --git a/filesystem/etc/calico/confd/templates/bird_aggr.cfg.template b/filesystem/etc/calico/confd/templates/bird_aggr.cfg.template new file mode 100644 index 000000000..e8e3075f9 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird_aggr.cfg.template @@ -0,0 +1,22 @@ +# Generated by confd +# ------------- Static black hole addresses ------------- +{{if ls "/"}} +protocol static { +{{range ls "/"}} +{{$parts := split . "-"}} +{{$cidr := join $parts "/"}} + route {{$cidr}} blackhole; +{{end}} +} +{{else}}# No static routes configured.{{end}} + +# Aggregation of routes on this host; export the block, nothing beneath it. +function calico_aggr () +{ +{{range ls "/"}} +{{$parts := split . "-"}} +{{$cidr := join $parts "/"}} + if ( net = {{$cidr}} ) then { accept; } + if ( net ~ {{$cidr}} ) then { reject; } +{{end}} +} diff --git a/filesystem/etc/calico/confd/templates/bird_aggr.toml.template b/filesystem/etc/calico/confd/templates/bird_aggr.toml.template new file mode 100644 index 000000000..b6a4dbf16 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird_aggr.toml.template @@ -0,0 +1,8 @@ +[template] +src = "bird_aggr.cfg.template" +dest = "/etc/calico/confd/config/bird_aggr.cfg" +prefix = "/calico/ipam/v2/host/NODENAME/ipv4/block" +keys = [ + "/", +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/templates/bird_ipam.cfg.template b/filesystem/etc/calico/confd/templates/bird_ipam.cfg.template new file mode 100644 index 000000000..482123b05 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird_ipam.cfg.template @@ -0,0 +1,32 @@ +# Generated by confd +filter calico_pools { + calico_aggr(); + custom_filters(); +{{range ls "/v1/ipam/v4/pool"}}{{$data := json (getv (printf "/v1/ipam/v4/pool/%s" .))}} + if ( net ~ {{$data.cidr}} ) then { + accept; + } +{{end}} + reject; +} + +{{$network_key := printf "/bgp/v1/host/%s/network_v4" (getenv "NODENAME")}}{{$network := getv $network_key}} +filter calico_ipip { +{{range ls "/v1/ipam/v4/pool"}}{{$data := json (getv (printf "/v1/ipam/v4/pool/%s" .))}} + if ( net ~ {{$data.cidr}} ) then { +{{if $data.ipip_mode}}{{if eq $data.ipip_mode "cross-subnet"}} + if ( from ~ {{$network}} ) then + krt_tunnel = ""; {{/* Destination in ipPool, mode is cross sub-net, route from-host on subnet, do not use IPIP */}} + else + krt_tunnel = "{{$data.ipip}}"; {{/* Destination in ipPool, mode is cross sub-net, route from-host off subnet, set the tunnel (if IPIP not enabled, value will be "") */}} + accept; + } {{else}} + krt_tunnel = "{{$data.ipip}}"; {{/* Destination in ipPool, mode not cross sub-net, set the tunnel (if IPIP not enabled, value will be "") */}} + accept; + } {{end}} {{else}} + krt_tunnel = "{{$data.ipip}}"; {{/* Destination in ipPool, mode field is not present, set the tunnel (if IPIP not enabled, value will be "") */}} + accept; + } {{end}} +{{end}} + accept; {{/* Destination is not in any ipPool, accept */}} +} diff --git a/filesystem/etc/calico/confd/templates/bird_ipam.toml.template b/filesystem/etc/calico/confd/templates/bird_ipam.toml.template new file mode 100644 index 000000000..404085ee9 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/bird_ipam.toml.template @@ -0,0 +1,9 @@ +[template] +src = "bird_ipam.cfg.template" +dest = "/etc/calico/confd/config/bird_ipam.cfg" +prefix = "/calico" +keys = [ + "/v1/ipam/v4/pool", + "/bgp/v1/host/NODENAME" +] +reload_cmd = "pkill -HUP bird || true" diff --git a/filesystem/etc/calico/confd/templates/custom_filters.cfg.template b/filesystem/etc/calico/confd/templates/custom_filters.cfg.template new file mode 100644 index 000000000..842aa0b3e --- /dev/null +++ b/filesystem/etc/calico/confd/templates/custom_filters.cfg.template @@ -0,0 +1,7 @@ +# Generated by confd +function custom_filters () +{ +{{range ls "/v4"}}{{$data := getv (printf "/v4/%s" .)}} +{{ $data }} +{{end}} +} diff --git a/filesystem/etc/calico/confd/templates/custom_filters6.cfg.template b/filesystem/etc/calico/confd/templates/custom_filters6.cfg.template new file mode 100644 index 000000000..150a4ae3b --- /dev/null +++ b/filesystem/etc/calico/confd/templates/custom_filters6.cfg.template @@ -0,0 +1,7 @@ +# Generated by confd +function custom_filters () +{ +{{range ls "/v6"}}{{$data := getv (printf "/v6/%s" .)}} +{{ $data }} +{{end}} +} diff --git a/filesystem/etc/calico/confd/templates/tunl-ip.template b/filesystem/etc/calico/confd/templates/tunl-ip.template new file mode 100644 index 000000000..5f78b77b0 --- /dev/null +++ b/filesystem/etc/calico/confd/templates/tunl-ip.template @@ -0,0 +1,7 @@ +We must dump all pool data to this file to trigger a resync. +Otherwise, confd notices the file hasn't changed and won't +run our python update script. + +{{range ls "/pool"}}{{$data := json (getv (printf "/pool/%s" .))}} + {{if $data.ipip}}{{if not $data.disabled}}{{$data.cidr}}{{end}}{{end}} +{{end}} diff --git a/filesystem/etc/calico/felix.cfg b/filesystem/etc/calico/felix.cfg new file mode 100644 index 000000000..269ea88b1 --- /dev/null +++ b/filesystem/etc/calico/felix.cfg @@ -0,0 +1,5 @@ +[global] +MetadataAddr = None +LogFilePath = None +LogSeverityFile = None +LogSeveritySys = None diff --git a/filesystem/etc/rc.local b/filesystem/etc/rc.local new file mode 100755 index 000000000..f2d67467d --- /dev/null +++ b/filesystem/etc/rc.local @@ -0,0 +1,103 @@ +# Handle old CALICO_NETWORKING environment by converting to the new config. +if [ -n "$CALICO_NETWORKING" ]; then + echo 'WARNING: $CALICO_NETWORKING will be deprecated: use $CALICO_NETWORKING_BACKEND instead' + if [ "$CALICO_NETWORKING" == "false" ]; then + export CALICO_NETWORKING_BACKEND=none + else + export CALICO_NETWORKING_BACKEND=bird + fi +fi + +# Run the startup initialisation script. These ensure the node is correctly +# configured to run. +startup || exit 1 + +# Source any additional environment that was added by the startup script. This +# is done immediately after the startup script because the environments are +# required by the remaining processing. +. startup.env + +# If possible pre-allocate the IP address on the IPIP tunnel. +allocate-ipip-addr || exit 1 + +# Create a directly to put enabled service files +mkdir /etc/service/enabled + +# XXX: Here and below we do all manupulations on /etc/service avoiding rm'ing +# dirs contained in Docker image. This is due to bug in Docker with graphdriver +# overlay on CentOS 7.X kernels (https://github.com/docker/docker/issues/15314) + +# Allow felix to be disabled, for example, if the user is running Felix +# outside the container. +if [ -z "$CALICO_DISABLE_FELIX" ]; then + cp -a /etc/service/available/felix /etc/service/enabled/ +fi + +case "$CALICO_NETWORKING_BACKEND" in + "none" ) + # If running in policy only mode, we don't need to run BIRD / Confd. + echo "CALICO_NETWORKING_BACKEND is none - no BGP daemon running" + ;; + "gobgp" ) + # Run calico-bgp-daemon instead of BIRD / Confd. + echo "CALICO_NETWORKING_BACKEND is gobgp - run calico-bgp-daemon" + cp -a /etc/service/available/calico-bgp-daemon /etc/service/enabled/ + sh -c 'for file in `find /etc/calico/confd/conf.d/ -not -name 'tunl-ip.toml' -type f`; do rm $file; done' + cp -a /etc/service/available/confd /etc/service/enabled/ + ;; + * ) + # Run BIRD / Confd. + # + # Run Confd in onetime mode, to ensure that we have a working config in place to allow bird(s) and + # felix to start. Don't fail startup if this confd execution fails. + # + # First generate the BIRD aggregation TOML file from the template by + # switching out the nodename. + sed "s/NODENAME/$NODENAME/" /etc/calico/confd/templates/bird6_aggr.toml.template > /etc/calico/confd/conf.d/bird6_aggr.toml + sed "s/NODENAME/$NODENAME/" /etc/calico/confd/templates/bird_aggr.toml.template > /etc/calico/confd/conf.d/bird_aggr.toml + sed "s/NODENAME/$NODENAME/" /etc/calico/confd/templates/bird_ipam.toml.template > /etc/calico/confd/conf.d/bird_ipam.toml + + # Run confd twice. Our confd TOML files are also generated from confd, so + # running twice ensures our starting configuration is correct. + if [ "$DATASTORE_TYPE" = "kubernetes" ]; then + confd -onetime -backend=k8s -confdir=/etc/calico/confd -log-level=debug -keep-stage-file >/felix-startup-1.log 2>&1 || true + confd -onetime -backend=k8s -confdir=/etc/calico/confd -log-level=debug -keep-stage-file >/felix-startup-2.log 2>&1 || true + else + # Use ETCD_ENDPOINTS in preferences to ETCD_AUTHORITY + ETCD_NODE=${ETCD_ENDPOINTS:=${ETCD_SCHEME:=http}://${ETCD_AUTHORITY}} + + # confd needs a "-node" arguments for each etcd endpoint. + ETCD_ENDPOINTS_CONFD=`echo "-node=$ETCD_NODE" | sed -e 's/,/ -node=/g'` + + confd -confdir=/etc/calico/confd -onetime ${ETCD_ENDPOINTS_CONFD} \ + -client-key=${ETCD_KEY_FILE} -client-cert=${ETCD_CERT_FILE} \ + -client-ca-keys=${ETCD_CA_CERT_FILE} -keep-stage-file >/felix-startup-1.log 2>&1 || true + confd -confdir=/etc/calico/confd -onetime ${ETCD_ENDPOINTS_CONFD} \ + -client-key=${ETCD_KEY_FILE} -client-cert=${ETCD_CERT_FILE} \ + -client-ca-keys=${ETCD_CA_CERT_FILE} -keep-stage-file >/felix-startup-2.log 2>&1 || true + fi + + # Enable the confd and bird services + cp -a /etc/service/available/bird /etc/service/enabled/ + cp -a /etc/service/available/bird6 /etc/service/enabled/ + cp -a /etc/service/available/confd /etc/service/enabled/ + ;; +esac + +# If running libnetwork plugin in a separate container, CALICO_LIBNETWORK_ENABLED would be false. +# CALICO_LIBNETWORK_ENABLED is "false" by default. It can be set by passing `--libnetwork` flag while starting the calico/node via calicoctl +if [ "$CALICO_LIBNETWORK_ENABLED" == "true" ]; then + echo "Starting libnetwork service" + cp -a /etc/service/available/libnetwork /etc/service/enabled/ +fi + +if [ "$CALICO_DISABLE_FILE_LOGGING" == "true" ]; then + rm -rf /etc/service/enabled/bird/log + rm -rf /etc/service/enabled/bird6/log + rm -rf /etc/service/enabled/confd/log + rm -rf /etc/service/enabled/felix/log + rm -rf /etc/service/enabled/libnetwork/log + rm -rf /etc/service/enabled/calico-bgp-daemon/log +fi + +echo "Calico node started successfully" diff --git a/filesystem/etc/service/available/bird/log/run b/filesystem/etc/service/available/bird/log/run new file mode 100755 index 000000000..fa6316930 --- /dev/null +++ b/filesystem/etc/service/available/bird/log/run @@ -0,0 +1,5 @@ +#!/bin/sh +LOGDIR=/var/log/calico/bird +mkdir -p $LOGDIR +# Prefix each line with a timestamp +exec svlogd -tt $LOGDIR \ No newline at end of file diff --git a/filesystem/etc/service/available/bird/run b/filesystem/etc/service/available/bird/run new file mode 100755 index 000000000..73122ae56 --- /dev/null +++ b/filesystem/etc/service/available/bird/run @@ -0,0 +1,3 @@ +#!/bin/sh +exec 2>&1 +exec bird -R -s /var/run/calico/bird.ctl -d -c /etc/calico/confd/config/bird.cfg \ No newline at end of file diff --git a/filesystem/etc/service/available/bird6/log/run b/filesystem/etc/service/available/bird6/log/run new file mode 100755 index 000000000..06071de34 --- /dev/null +++ b/filesystem/etc/service/available/bird6/log/run @@ -0,0 +1,5 @@ +#!/bin/sh +LOGDIR=/var/log/calico/bird6 +mkdir -p $LOGDIR +# Prefix each line with a timestamp +exec svlogd -tt $LOGDIR \ No newline at end of file diff --git a/filesystem/etc/service/available/bird6/run b/filesystem/etc/service/available/bird6/run new file mode 100755 index 000000000..eed06fb56 --- /dev/null +++ b/filesystem/etc/service/available/bird6/run @@ -0,0 +1,3 @@ +#!/bin/sh +exec 2>&1 +exec bird6 -R -s /var/run/calico/bird6.ctl -d -c /etc/calico/confd/config/bird6.cfg \ No newline at end of file diff --git a/filesystem/etc/service/available/calico-bgp-daemon/log/run b/filesystem/etc/service/available/calico-bgp-daemon/log/run new file mode 100755 index 000000000..0fc6cda04 --- /dev/null +++ b/filesystem/etc/service/available/calico-bgp-daemon/log/run @@ -0,0 +1,4 @@ +#!/bin/sh +LOGDIR=/var/log/calico/calico-bgp-daemon +mkdir -p $LOGDIR +exec svlogd $LOGDIR diff --git a/filesystem/etc/service/available/calico-bgp-daemon/run b/filesystem/etc/service/available/calico-bgp-daemon/run new file mode 100755 index 000000000..c84baff0c --- /dev/null +++ b/filesystem/etc/service/available/calico-bgp-daemon/run @@ -0,0 +1,3 @@ +#!/bin/sh +exec 2>&1 +exec calico-bgp-daemon diff --git a/filesystem/etc/service/available/confd/log/run b/filesystem/etc/service/available/confd/log/run new file mode 100755 index 000000000..a50e86587 --- /dev/null +++ b/filesystem/etc/service/available/confd/log/run @@ -0,0 +1,4 @@ +#!/bin/sh +LOGDIR=/var/log/calico/confd +mkdir -p $LOGDIR +exec svlogd $LOGDIR \ No newline at end of file diff --git a/filesystem/etc/service/available/confd/run b/filesystem/etc/service/available/confd/run new file mode 100755 index 000000000..d18ac0962 --- /dev/null +++ b/filesystem/etc/service/available/confd/run @@ -0,0 +1,14 @@ +#!/bin/sh +exec 2>&1 + +if [ "$DATASTORE_TYPE" = "kubernetes" ] +then + exec confd -confdir=/etc/calico/confd -interval=5 -log-level=debug -backend=k8s +else + ETCD_NODE=${ETCD_ENDPOINTS:=${ETCD_SCHEME:=http}://${ETCD_AUTHORITY}} + ETCD_ENDPOINTS_CONFD=`echo "-node=$ETCD_NODE" | sed -e 's/,/ -node=/g'` + + exec confd -confdir=/etc/calico/confd -interval=5 -watch --log-level=debug \ + $ETCD_ENDPOINTS_CONFD -client-key=${ETCD_KEY_FILE} \ + -client-cert=${ETCD_CERT_FILE} -client-ca-keys=${ETCD_CA_CERT_FILE} +fi diff --git a/filesystem/etc/service/available/felix/log/run b/filesystem/etc/service/available/felix/log/run new file mode 100755 index 000000000..1dfec64dd --- /dev/null +++ b/filesystem/etc/service/available/felix/log/run @@ -0,0 +1,4 @@ +#!/bin/sh +LOGDIR=/var/log/calico/felix +mkdir -p $LOGDIR +exec svlogd $LOGDIR \ No newline at end of file diff --git a/filesystem/etc/service/available/felix/run b/filesystem/etc/service/available/felix/run new file mode 100755 index 000000000..6b1be8df8 --- /dev/null +++ b/filesystem/etc/service/available/felix/run @@ -0,0 +1,18 @@ +#!/bin/sh +exec 2>&1 +# Felix doesn't understand NODENAME, but the container exports it as a common +# interface. This ensures Felix gets the right name for the node. +if [ ! -z $NODENAME ]; then + export FELIX_FELIXHOSTNAME=$NODENAME +fi +export FELIX_ETCDADDR=$ETCD_AUTHORITY +export FELIX_ETCDENDPOINTS=$ETCD_ENDPOINTS +export FELIX_ETCDSCHEME=$ETCD_SCHEME +export FELIX_ETCDCAFILE=$ETCD_CA_CERT_FILE +export FELIX_ETCDKEYFILE=$ETCD_KEY_FILE +export FELIX_ETCDCERTFILE=$ETCD_CERT_FILE +# Felix hangs if DATASTORETYPE is empty: see projectcalico/felix issue #1156. +if [ ! -z $DATASTORE_TYPE ]; then + export FELIX_DATASTORETYPE=$DATASTORE_TYPE +fi +exec calico-felix diff --git a/filesystem/etc/service/available/libnetwork/log/run b/filesystem/etc/service/available/libnetwork/log/run new file mode 100755 index 000000000..4ad0f626a --- /dev/null +++ b/filesystem/etc/service/available/libnetwork/log/run @@ -0,0 +1,4 @@ +#!/bin/sh +LOGDIR=/var/log/calico/libnetwork +mkdir -p $LOGDIR +exec svlogd $LOGDIR diff --git a/filesystem/etc/service/available/libnetwork/run b/filesystem/etc/service/available/libnetwork/run new file mode 100755 index 000000000..61636064e --- /dev/null +++ b/filesystem/etc/service/available/libnetwork/run @@ -0,0 +1,3 @@ +#!/bin/sh +exec 2>&1 +exec libnetwork-plugin diff --git a/filesystem/sbin/restart-calico-confd b/filesystem/sbin/restart-calico-confd new file mode 100755 index 000000000..c3c7e89bd --- /dev/null +++ b/filesystem/sbin/restart-calico-confd @@ -0,0 +1,2 @@ +#!/bin/sh +sv restart /etc/service/enabled/confd diff --git a/filesystem/sbin/start_runit b/filesystem/sbin/start_runit new file mode 100755 index 000000000..385088df1 --- /dev/null +++ b/filesystem/sbin/start_runit @@ -0,0 +1,16 @@ +#!/bin/sh +# From https://github.com/faisyl/alpine-runit +env > /etc/envvars + +/etc/rc.local +retval=$? +if [ $retval -ne 0 ]; +then + echo >&2 "Calico node failed to start" + exit $retval +fi + +# Source any additional environment that was added by the startup script +. startup.env + +exec /sbin/runsvdir -P /etc/service/enabled diff --git a/filesystem/sbin/versions b/filesystem/sbin/versions new file mode 100755 index 000000000..fc51eac7c --- /dev/null +++ b/filesystem/sbin/versions @@ -0,0 +1,10 @@ +#!/bin/sh +confd -version +bird --version +echo -n "libnetwork-plugin: " +libnetwork-plugin -v +echo -n "calico-bgp-daemon: " +calico-bgp-daemon -v +echo "" +echo "Felix:" +calico-felix --version diff --git a/glide.lock b/glide.lock new file mode 100644 index 000000000..3da1a18a6 --- /dev/null +++ b/glide.lock @@ -0,0 +1,325 @@ +hash: d193d13ff5e3b54ec6c158bf97bd470b81f295ef95bf602fe79813d706d0e597 +updated: 2017-06-16T22:20:48.977274261Z +imports: +- name: github.com/coreos/etcd + version: 17ae440991da3bdb2df4309936dd2074f66ec394 + subpackages: + - client + - pkg/pathutil + - pkg/tlsutil + - pkg/transport + - pkg/types + - version +- name: github.com/coreos/go-semver + version: 568e959cd89871e61434c1143528d9162da89ef2 + subpackages: + - semver +- name: github.com/davecgh/go-spew + version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d + subpackages: + - spew +- name: github.com/docker/distribution + version: cd27f179f2c10c5d300e6d09025b538c475b0d51 + subpackages: + - digest + - reference +- name: github.com/emicklei/go-restful + version: 09691a3b6378b740595c1002f40c34dd5f218a22 + subpackages: + - log + - swagger +- name: github.com/ghodss/yaml + version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee +- name: github.com/go-openapi/jsonpointer + version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 +- name: github.com/go-openapi/jsonreference + version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 +- name: github.com/go-openapi/spec + version: 6aced65f8501fe1217321abf0749d354824ba2ff +- name: github.com/go-openapi/swag + version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 +- name: github.com/gogo/protobuf + version: 909568be09de550ed094403c2bf8a261b5bb730a + subpackages: + - proto + - sortkeys +- name: github.com/golang/glog + version: 44145f04b68cf362d9c4df2182967c2275eaefed +- name: github.com/google/gofuzz + version: 44d81051d367757e1c7c6a5a86423ece9afcf63c +- name: github.com/howeyc/gopass + version: 3ca23474a7c7203e0a0a070fd33508f6efdb9b3d +- name: github.com/imdario/mergo + version: 6633656539c1639d9d78127b7d47c622b5d7b6dc +- name: github.com/juju/ratelimit + version: 77ed1c8a01217656d2080ad51981f6e99adaa177 +- name: github.com/kelseyhightower/envconfig + version: 8bf4bbfc795e2c7c8a5ea47b707453ed019e2ad4 +- name: github.com/mailru/easyjson + version: d5b7844b561a7bc640052f1b935f7b800330d7e0 + subpackages: + - buffer + - jlexer + - jwriter +- name: github.com/onsi/ginkgo + version: f40a49d81e5c12e90400620b6242fb29a8e7c9d9 + subpackages: + - config + - extensions/table + - internal/codelocation + - internal/containernode + - internal/failer + - internal/leafnodes + - internal/remote + - internal/spec + - internal/spec_iterator + - internal/specrunner + - internal/suite + - internal/testingtproxy + - internal/writer + - reporters + - reporters/stenographer + - reporters/stenographer/support/go-colorable + - reporters/stenographer/support/go-isatty + - types +- name: github.com/projectcalico/go-json + version: 6219dc7339ba20ee4c57df0a8baac62317d19cb1 + subpackages: + - json +- name: github.com/projectcalico/go-yaml + version: 955bc3e451ef0c9df8b9113bf2e341139cdafab2 +- name: github.com/projectcalico/go-yaml-wrapper + version: 598e54215bee41a19677faa4f0c32acd2a87eb56 +- name: github.com/projectcalico/libcalico-go + version: e40a090d7d025f7ab60f7a5e5bc362f875116754 + subpackages: + - lib/api + - lib/api/unversioned + - lib/backend + - lib/backend/api + - lib/backend/compat + - lib/backend/etcd + - lib/backend/k8s + - lib/backend/k8s/resources + - lib/backend/k8s/thirdparty + - lib/backend/model + - lib/client + - lib/converter + - lib/errors + - lib/hash + - lib/hwm + - lib/ipip + - lib/net + - lib/numorstring + - lib/scope + - lib/selector + - lib/selector/parser + - lib/selector/tokenizer + - lib/testutils + - lib/validator +- name: github.com/PuerkitoBio/purell + version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 +- name: github.com/PuerkitoBio/urlesc + version: 5bd2802263f21d8788851d5305584c82a5c75d7e +- name: github.com/satori/go.uuid + version: b061729afc07e77a8aa4fad0a2fd840958f1942a +- name: github.com/Sirupsen/logrus + version: ba1b36c82c5e05c4f912a88eab0dcd91a171688f +- name: github.com/spf13/pflag + version: 08b1a584251b5b62f458943640fc8ebd4d50aaa5 +- name: github.com/ugorji/go + version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74 + subpackages: + - codec +- name: golang.org/x/crypto + version: 1351f936d976c60a0a48d728281922cf63eafb8d + subpackages: + - bcrypt + - blowfish + - ssh/terminal +- name: golang.org/x/net + version: f2499483f923065a842d38eb4c7f1927e6fc6e6d + subpackages: + - context + - http2 + - http2/hpack + - idna + - lex/httplex +- name: golang.org/x/sys + version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 + subpackages: + - unix +- name: golang.org/x/text + version: 19e51611da83d6be54ddafce4a4af510cb3e9ea4 + subpackages: + - cases + - internal + - internal/tag + - language + - runes + - secure/bidirule + - secure/precis + - transform + - unicode/bidi + - unicode/norm + - width +- name: gopkg.in/go-playground/validator.v8 + version: 5f57d2222ad794d0dffb07e664ea05e2ee07d60c +- name: gopkg.in/inf.v0 + version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 +- name: gopkg.in/tchap/go-patricia.v2 + version: 666120de432aea38ab06bd5c818f04f4129882c9 + subpackages: + - patricia +- name: gopkg.in/yaml.v2 + version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 +- name: k8s.io/apimachinery + version: b317fa7ec8e0e7d1f77ac63bf8c3ec7b29a2a215 + subpackages: + - pkg/api/errors + - pkg/api/meta + - pkg/api/resource + - pkg/apimachinery + - pkg/apimachinery/announced + - pkg/apimachinery/registered + - pkg/apis/meta/v1 + - pkg/apis/meta/v1/unstructured + - pkg/conversion + - pkg/conversion/queryparams + - pkg/fields + - pkg/labels + - pkg/openapi + - pkg/runtime + - pkg/runtime/schema + - pkg/runtime/serializer + - pkg/runtime/serializer/json + - pkg/runtime/serializer/protobuf + - pkg/runtime/serializer/recognizer + - pkg/runtime/serializer/streaming + - pkg/runtime/serializer/versioning + - pkg/selection + - pkg/types + - pkg/util/diff + - pkg/util/errors + - pkg/util/framer + - pkg/util/intstr + - pkg/util/json + - pkg/util/net + - pkg/util/rand + - pkg/util/runtime + - pkg/util/sets + - pkg/util/validation + - pkg/util/validation/field + - pkg/util/wait + - pkg/util/yaml + - pkg/version + - pkg/watch + - third_party/forked/golang/reflect +- name: k8s.io/client-go + version: 4a3ab2f5be5177366f8206fd79ce55ca80e417fa + subpackages: + - discovery + - kubernetes + - kubernetes/scheme + - kubernetes/typed/apps/v1beta1 + - kubernetes/typed/authentication/v1 + - kubernetes/typed/authentication/v1beta1 + - kubernetes/typed/authorization/v1 + - kubernetes/typed/authorization/v1beta1 + - kubernetes/typed/autoscaling/v1 + - kubernetes/typed/autoscaling/v2alpha1 + - kubernetes/typed/batch/v1 + - kubernetes/typed/batch/v2alpha1 + - kubernetes/typed/certificates/v1beta1 + - kubernetes/typed/core/v1 + - kubernetes/typed/extensions/v1beta1 + - kubernetes/typed/policy/v1beta1 + - kubernetes/typed/rbac/v1alpha1 + - kubernetes/typed/rbac/v1beta1 + - kubernetes/typed/settings/v1alpha1 + - kubernetes/typed/storage/v1 + - kubernetes/typed/storage/v1beta1 + - pkg/api + - pkg/api/errors + - pkg/api/install + - pkg/api/meta + - pkg/api/unversioned + - pkg/api/v1 + - pkg/apis/apps + - pkg/apis/apps/install + - pkg/apis/apps/v1beta1 + - pkg/apis/authentication + - pkg/apis/authentication/install + - pkg/apis/authentication/v1 + - pkg/apis/authentication/v1beta1 + - pkg/apis/authorization + - pkg/apis/authorization/install + - pkg/apis/authorization/v1 + - pkg/apis/authorization/v1beta1 + - pkg/apis/autoscaling + - pkg/apis/autoscaling/install + - pkg/apis/autoscaling/v1 + - pkg/apis/autoscaling/v2alpha1 + - pkg/apis/batch + - pkg/apis/batch/install + - pkg/apis/batch/v1 + - pkg/apis/batch/v2alpha1 + - pkg/apis/certificates + - pkg/apis/certificates/install + - pkg/apis/certificates/v1beta1 + - pkg/apis/extensions + - pkg/apis/extensions/install + - pkg/apis/extensions/v1beta1 + - pkg/apis/policy + - pkg/apis/policy/install + - pkg/apis/policy/v1beta1 + - pkg/apis/rbac + - pkg/apis/rbac/install + - pkg/apis/rbac/v1alpha1 + - pkg/apis/rbac/v1beta1 + - pkg/apis/settings + - pkg/apis/settings/install + - pkg/apis/settings/v1alpha1 + - pkg/apis/storage + - pkg/apis/storage/install + - pkg/apis/storage/v1 + - pkg/apis/storage/v1beta1 + - pkg/fields + - pkg/runtime + - pkg/runtime/schema + - pkg/runtime/serializer + - pkg/util + - pkg/util/parsers + - pkg/util/wait + - pkg/version + - pkg/watch + - rest + - rest/watch + - tools/auth + - tools/cache + - tools/clientcmd + - tools/clientcmd/api + - tools/clientcmd/api/latest + - tools/clientcmd/api/v1 + - tools/metrics + - transport + - util/cert + - util/clock + - util/flowcontrol + - util/homedir + - util/integer +testImports: +- name: github.com/onsi/gomega + version: 9b8c753e8dfb382618ba8fa19b4197b5dcb0434c + subpackages: + - format + - internal/assertion + - internal/asyncassertion + - internal/oraclematcher + - internal/testingtsupport + - matchers + - matchers/support/goraph/bipartitegraph + - matchers/support/goraph/edge + - matchers/support/goraph/node + - matchers/support/goraph/util + - types diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 000000000..4a43bcdd5 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,18 @@ +package: github.com/projectcalico/calico/calico_node +import: +- package: github.com/Sirupsen/logrus + version: ^0.10.0 +- package: github.com/projectcalico/libcalico-go + version: ^1.4.4 + subpackages: + - lib/api + - lib/client + - lib/errors + - lib/ipip + - lib/net + - lib/numorstring +testImport: +- package: github.com/onsi/ginkgo + subpackages: + - extensions/table +- package: github.com/onsi/gomega diff --git a/startup/autodetection/autodetection_suite_test.go b/startup/autodetection/autodetection_suite_test.go new file mode 100644 index 000000000..cee6e8a65 --- /dev/null +++ b/startup/autodetection/autodetection_suite_test.go @@ -0,0 +1,19 @@ +package autodetection_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" + + "github.com/projectcalico/libcalico-go/lib/testutils" +) + +func init() { + testutils.HookLogrusForGinkgo() +} + +func TestCommands(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Autodetection Suite") +} diff --git a/startup/autodetection/filtered.go b/startup/autodetection/filtered.go new file mode 100644 index 000000000..68de5df3e --- /dev/null +++ b/startup/autodetection/filtered.go @@ -0,0 +1,51 @@ +// Copyright (c) 2016 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autodetection + +import ( + "errors" + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/projectcalico/libcalico-go/lib/net" +) + +// FilteredEnumeration performs basic IP and IPNetwork discovery by enumerating +// all interfaces and filtering in/out based on the supplied filter regex. +// +// The incl and excl slice of regex strings may be nil. +func FilteredEnumeration(incl, excl []string, version int) (*Interface, *net.IPNet, error) { + interfaces, err := GetInterfaces(incl, excl, version) + if err != nil { + return nil, nil, err + } + if len(interfaces) == 0 { + return nil, nil, errors.New("no valid host interfaces found") + } + + // Find the first interface with a valid IP address and network. + // We initialise the IP with the first valid IP that we find just in + // case we don't find an IP *and* network. + for _, i := range interfaces { + log.WithField("Name", i.Name).Debug("Check interface") + for _, c := range i.Cidrs { + log.WithField("CIDR", c).Debug("Check address") + if c.IP.IsGlobalUnicast() { + return &i, &c, nil + } + } + } + + return nil, nil, fmt.Errorf("no valid IPv%d addresses found on the host interfaces", version) +} diff --git a/startup/autodetection/filtered_test.go b/startup/autodetection/filtered_test.go new file mode 100644 index 000000000..7e6e6ea18 --- /dev/null +++ b/startup/autodetection/filtered_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2016 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autodetection_test + +import ( + "github.com/projectcalico/calico/calico_node/startup/autodetection" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Filtered enumeration tests", func() { + + Describe("No filters", func() { + Context("Get interface and address", func() { + + iface, addr, err := autodetection.FilteredEnumeration(nil, nil, 4) + It("should have enumerated at least one IP address", func() { + Expect(err).To(BeNil()) + Expect(iface).ToNot(BeNil()) + Expect(addr).ToNot(BeNil()) + }) + }) + }) +}) diff --git a/startup/autodetection/interfaces.go b/startup/autodetection/interfaces.go new file mode 100644 index 000000000..fe9d88fc5 --- /dev/null +++ b/startup/autodetection/interfaces.go @@ -0,0 +1,104 @@ +// Copyright (c) 2016 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autodetection + +import ( + "net" + "regexp" + "strings" + + log "github.com/Sirupsen/logrus" + cnet "github.com/projectcalico/libcalico-go/lib/net" +) + +// Interface contains details about an interface on the host. +type Interface struct { + Name string + Cidrs []cnet.IPNet +} + +// GetInterfaces returns a list of all interfaces, skipping any interfaces whose +// name matches any of the exclusion list regexes, and including those on the +// inclusion list. +func GetInterfaces(includeRegexes []string, excludeRegexes []string, version int) ([]Interface, error) { + netIfaces, err := net.Interfaces() + if err != nil { + log.WithError(err).Warnf("Failed to enumerate interfaces") + return nil, err + } + + var filteredIfaces []Interface + var includeRegexp *regexp.Regexp + var excludeRegexp *regexp.Regexp + + // Create single include and exclude regexes to perform the interface + // check. + if len(includeRegexes) > 0 { + if includeRegexp, err = regexp.Compile("(" + strings.Join(includeRegexes, ")|(") + ")"); err != nil { + log.WithError(err).Warnf("Invalid interface regex") + return nil, err + } + } + if len(excludeRegexes) > 0 { + if excludeRegexp, err = regexp.Compile("(" + strings.Join(excludeRegexes, ")|(") + ")"); err != nil { + log.WithError(err).Warnf("Invalid interface regex") + return nil, err + } + } + + // Loop through interfaces filtering on the regexes. Loop in reverse + // order to maintain behavior with older versions. + for idx := len(netIfaces)-1; idx >= 0; idx-- { + iface := netIfaces[idx] + include := (includeRegexp == nil) || includeRegexp.MatchString(iface.Name) + exclude := (excludeRegexp != nil) && excludeRegexp.MatchString(iface.Name) + if include && !exclude { + if i, err := convertInterface(&iface, version); err == nil { + filteredIfaces = append(filteredIfaces, *i) + } + } + } + return filteredIfaces, nil +} + +// convertInterface converts a net.Interface to our Interface type (which has +// converted address types). +func convertInterface(i *net.Interface, version int) (*Interface, error) { + log.WithField("Interface", i.Name).Debug("Querying interface addresses") + addrs, err := i.Addrs() + if err != nil { + log.Warnf("Cannot get interface address(es): %v", err) + return nil, err + } + + iface := &Interface{Name: i.Name} + for _, addr := range addrs { + addrStr := addr.String() + ip, ipNet, err := cnet.ParseCIDR(addrStr) + if err != nil { + log.WithError(err).WithField("Address", addrStr).Warning("Failed to parse CIDR") + continue + } + + if ip.Version() == version { + // Switch out the IP address in the network with the + // interface IP to get the CIDR (IP + network). + ipNet.IP = ip.IP + log.WithField("CIDR", ipNet).Debug("Found valid IP address and network") + iface.Cidrs = append(iface.Cidrs, *ipNet) + } + } + + return iface, nil +} diff --git a/startup/autodetection/reachaddr.go b/startup/autodetection/reachaddr.go new file mode 100644 index 000000000..8ef6dd7e2 --- /dev/null +++ b/startup/autodetection/reachaddr.go @@ -0,0 +1,66 @@ +// Copyright (c) 2017 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package autodetection + +import ( + "fmt" + gonet "net" + + log "github.com/Sirupsen/logrus" + "github.com/projectcalico/libcalico-go/lib/net" +) + +// ReachDestination auto-detects the interface Network by setting up a UDP +// connection to a "reach" destination. +func ReachDestination(dest string, version int) (*net.IPNet, error) { + log.Debugf("Auto-detecting IPv%d CIDR by reaching destination %s", version, dest) + + // Open a UDP connection to determine which external IP address is + // used to access the supplied destination. + protocol := fmt.Sprintf("udp%d", version) + address := fmt.Sprintf("[%s]:80", dest) + conn, err := gonet.Dial(protocol, address) + if err != nil { + return nil, err + } + defer conn.Close() + + // Get the local address as a golang IP and use that to find the matching + // interface CIDR. + addr := conn.LocalAddr() + if addr == nil { + return nil, fmt.Errorf("no address detected by connecting to %s", dest) + } + udpAddr := addr.(*gonet.UDPAddr) + log.WithFields(log.Fields{"IP": udpAddr.IP, "Destination": dest}).Info("Auto-detected address by connecting to remote") + + // Get a full list of interface and IPs and find the CIDR matching the + // found IP. + ifaces, err := GetInterfaces(nil, nil, version) + if err != nil { + return nil, err + } + for _, iface := range ifaces { + log.WithField("Name", iface.Name).Info("Checking interface CIDRs") + for _, cidr := range iface.Cidrs { + log.WithField("CIDR", cidr.String()).Info("Checking CIDR") + if cidr.IP.Equal(udpAddr.IP) { + log.WithField("CIDR", cidr.String()).Info("Found matching interface CIDR") + return &cidr, nil + } + } + } + + return nil, fmt.Errorf("autodetected IPv%d address does not match any addresses found on local interfaces: %s", version, udpAddr.IP.String()) +} diff --git a/startup/startup.go b/startup/startup.go new file mode 100644 index 000000000..ce9cf7623 --- /dev/null +++ b/startup/startup.go @@ -0,0 +1,779 @@ +// Copyright (c) 2016 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "strings" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/projectcalico/calico/calico_node/calicoclient" + "github.com/projectcalico/calico/calico_node/startup/autodetection" + "github.com/projectcalico/libcalico-go/lib/api" + "github.com/projectcalico/libcalico-go/lib/client" + "github.com/projectcalico/libcalico-go/lib/errors" + "github.com/projectcalico/libcalico-go/lib/ipip" + "github.com/projectcalico/libcalico-go/lib/net" + "github.com/projectcalico/libcalico-go/lib/numorstring" +) + +const ( + DEFAULT_IPV4_POOL_CIDR = "192.168.0.0/16" + DEFAULT_IPV6_POOL_CIDR = "fd80:24e2:f998:72d6::/64" + AUTODETECTION_METHOD_FIRST = "first-found" + AUTODETECTION_METHOD_CAN_REACH = "can-reach=" + AUTODETECTION_METHOD_INTERFACE = "interface=" +) + +// For testing purposes we define an exit function that we can override. +var exitFunction = os.Exit + +// This file contains the main startup processing for the calico/node. This +// includes: +// - Detecting IP address and Network to use for BGP +// - Configuring the node resource with IP/AS information provided in the +// environment, or autodetected. +// - Creating default IP Pools for quick-start use + +func main() { + // Determine the name for this node and ensure the environment is always + // available in the startup env file that is sourced in rc.local. + nodeName := determineNodeName() + + // Create the Calico API client. + cfg, client := calicoclient.CreateClient() + + // An explicit value of true is required to wait for the datastore. + if os.Getenv("WAIT_FOR_DATASTORE") == "true" { + waitForConnection(client) + log.Info("Datastore is ready") + } else { + message("Skipping datastore connection test") + } + + // Query the current Node resources. We update our node resource with + // updated IP data and use the full list of nodes for validation. + node := getNode(client, nodeName) + + // Configure and verify the node IP addresses and subnets. + configureIPsAndSubnets(node) + + // Configure the node AS number. + configureASNumber(node) + + // Check for conflicting node configuration + checkConflictingNodes(client, node) + + // Check expected filesystem + ensureFilesystemAsExpected() + + // Apply the updated node resource. + if _, err := client.Nodes().Apply(node); err != nil { + fatal("Unable to set node resource configuration: %s", err) + terminate() + } + + // Configure IP Pool configuration. + configureIPPools(client) + + // Set other Felix config that is not yet in the node resource. Skip for Kubernetes as + // the keys do not yet exist + if cfg.Spec.DatastoreType != api.Kubernetes { + if err := ensureDefaultConfig(client, node); err != nil { + fatal("Unable to set global default configuration: %s", err) + terminate() + } + } + + // Write the startup.env file now that we are ready to start other + // components. + writeStartupEnv(nodeName, node.Spec.BGP.IPv4Address, node.Spec.BGP.IPv6Address) + + // Tell the user what the name of the node is. + message("Using node name: %s", nodeName) +} + +// determineNodeName is called to determine the node name to use for this instance +// of calico/node. +func determineNodeName() string { + // Determine the name of this node. + nodeName := os.Getenv("NODENAME") + if nodeName == "" { + // NODENAME not specified, check HOSTNAME (we maintain this for + // backwards compatibility). + log.Info("NODENAME environment not specified - check HOSTNAME") + nodeName = os.Getenv("HOSTNAME") + } + if nodeName == "" { + // The node name has not been specified. We need to use the OS + // hostname - but should warn the user that this is not a + // recommended way to start the node container. + var err error + if nodeName, err = os.Hostname(); err != nil { + log.Info("Unable to determine hostname - exiting") + panic(err) + } + + message("******************************************************************************") + message("* WARNING *") + message("* Auto-detecting node name. It is recommended that an explicit fixed value *") + message("* is supplied using the NODENAME environment variable. Using a fixed value *") + message("* ensures that any changes to the compute host's hostname will not affect *") + message("* the Calico configuration when calico/node restarts. *") + message("******************************************************************************") + } + return nodeName +} + +// waitForConnection waits for the datastore to become accessible. +func waitForConnection(c *client.Client) { + message("Checking datastore connection") + for { + // Query some arbitrary configuration to see if the connection + // is working. Getting a specific Node is a good option, even + // if the Node does not exist. + _, err := c.Nodes().Get(api.NodeMetadata{Name: "foo"}) + + // We only care about a couple of error cases, all others would + // suggest the datastore is accessible. + if err != nil { + switch err.(type) { + case errors.ErrorConnectionUnauthorized: + fatal("Connection to the datastore is unauthorized") + terminate() + case errors.ErrorDatastoreError: + time.Sleep(1000 * time.Millisecond) + continue + } + } + + // We've connected to the datastore - break out of the loop. + break + } + message("Datastore connection verified") +} + +// writeStartupEnv writes out the startup.env file to set environment variables +// that are required by confd/bird etc. but may not have been passed into the +// container. +func writeStartupEnv(nodeName string, ip, ip6 *net.IPNet) { + text := "export NODENAME=" + nodeName + "\n" + + // TODO: See https://github.com/projectcalico/calico-bgp-daemon/issues/18 + // The following entries are required for go-bgp. Once updated to use + // NODENAME and the node IP parameters, these entries can be removed. + text += "export HOSTNAME=" + nodeName + "\n" + if ip != nil { + text += "export IP=" + ip.IP.String() + "\n" + } + if ip6 != nil { + text += "export IP6=" + ip6.IP.String() + "\n" + } + + // Write out the startup.env file to ensure required environments are + // set (which they might not otherwise be). + if err := ioutil.WriteFile("startup.env", []byte(text), 0666); err != nil { + log.WithError(err).Info("Unable to write to startup.env") + fatal("Unable to write to local filesystem") + terminate() + } +} + +// getNode returns the current node configuration. If this node has not yet +// been created, it returns a blank node resource. +func getNode(client *client.Client, nodeName string) *api.Node { + meta := api.NodeMetadata{Name: nodeName} + node, err := client.Nodes().Get(meta) + + if err != nil { + if _, ok := err.(errors.ErrorResourceDoesNotExist); !ok { + log.WithError(err).WithField("Name", nodeName).Info("Unable to query node configuration") + fatal("Unable to access datastore to query node configuration") + terminate() + } + + log.WithField("Name", nodeName).Info("Building new node resource") + node = &api.Node{Metadata: api.NodeMetadata{Name: nodeName}} + } + + return node +} + +// configureIPsAndSubnets updates the supplied node resource with IP and Subnet +// information to use for BGP. +func configureIPsAndSubnets(node *api.Node) { + // If the node resource currently has no BGP configuration, add an empty + // set of configuration as it makes the processing below easier, and we + // must end up configuring some BGP fields before we complete. + if node.Spec.BGP == nil { + log.Info("Initialise BGP data") + node.Spec.BGP = &api.NodeBGPSpec{} + } + + // Determine the autodetection type for IPv4 and IPv6. Note that we + // only autodetect IPv4 when it has not been specified. IPv6 must be + // explicitly requested using the "autodetect" value. + // + // If we aren't auto-detecting then we need to validate the configured + // value and possibly fix up missing subnet configuration. + ipv4Env := os.Getenv("IP") + if ipv4Env == "autodetect" || (ipv4Env == "" && node.Spec.BGP.IPv4Address == nil) { + adm := os.Getenv("IP_AUTODETECTION_METHOD") + cidr := autoDetectCIDR(adm, 4) + if cidr != nil { + // We autodetected an IPv4 address so update the value in the node. + node.Spec.BGP.IPv4Address = cidr + } else if node.Spec.BGP.IPv4Address == nil { + // No IPv4 address is configured, but we always require one, so exit. + fatal("Couldn't autodetect a management IPv4 address:") + message(" - provide an IPv4 address by configuring one in the node resource, or") + message(" - provide an IPv4 address using the IP environment, or") + message(" - if auto-detecting, use a different autodetection method.") + terminate() + } else { + // No IPv4 autodetected, but a previous one was configured. + // Tell the user we are leaving the value unchanged. We + // will validate that the IP matches one on the interface. + warning("Autodetection of IPv4 address failed, keeping existing value: %s", node.Spec.BGP.IPv4Address.String()) + validateIP(node.Spec.BGP.IPv4Address) + } + } else { + if ipv4Env != "" { + node.Spec.BGP.IPv4Address = parseIPEnvironment("IP", ipv4Env, 4) + } + validateIP(node.Spec.BGP.IPv4Address) + } + + ipv6Env := os.Getenv("IP6") + if ipv6Env == "autodetect" { + adm := os.Getenv("IP6_AUTODETECTION_METHOD") + cidr := autoDetectCIDR(adm, 6) + if cidr != nil { + // We autodetected an IPv6 address so update the value in the node. + node.Spec.BGP.IPv6Address = cidr + } else if node.Spec.BGP.IPv6Address == nil { + // No IPv6 address is configured, but we have requested one, so exit. + fatal("Couldn't autodetect a management IPv6 address:") + message(" - provide an IPv6 address by configuring one in the node resource, or") + message(" - provide an IPv6 address using the IP6 environment, or") + message(" - use a different autodetection method, or") + message(" - don't request autodetection of an IPv6 address.") + terminate() + } else { + // No IPv6 autodetected, but a previous one was configured. + // Tell the user we are leaving the value unchanged. We + // will validate that the IP matches one on the interface. + warning("Autodetection of IPv6 address failed, keeping existing value: %s", node.Spec.BGP.IPv6Address.String()) + validateIP(node.Spec.BGP.IPv4Address) + } + } else { + if ipv6Env != "" { + node.Spec.BGP.IPv6Address = parseIPEnvironment("IP6", ipv6Env, 6) + } + validateIP(node.Spec.BGP.IPv6Address) + } + +} + +// fetchAndValidateIPAndNetwork fetches and validates the IP configuration from +// either the environment variables or from the values already configured in the +// node. +func parseIPEnvironment(envName, envValue string, version int) *net.IPNet { + // To parse the environment (which could be an IP or a CIDR), convert + // to a JSON string and use the UnmarshalJSON method on the IPNet + // struct to parse the value. + ip := &net.IPNet{} + err := ip.UnmarshalJSON([]byte("\"" + envValue + "\"")) + if err != nil || ip.Version() != version { + fatal("Environment does not contain a valid IPv%d address: %s=%s", version, envName, envValue) + terminate() + } + message("Using IPv%d address from environment: %s=%s", ip.Version(), envName, envValue) + + return ip +} + +// validateIP checks that the IP address is actually on one of the host +// interfaces and warns if not. +func validateIP(ipn *net.IPNet) { + // No validation required if no IP address is specified. + if ipn == nil { + return + } + + // Pull out the IP as a net.IP (it has useful string and version methods). + ip := net.IP{ipn.IP} + + // Get a complete list of interfaces with their addresses and check if + // the IP address can be found. + ifaces, err := autodetection.GetInterfaces(nil, nil, ip.Version()) + if err != nil { + fatal("Unable to query host interfaces: %s", err) + terminate() + } + if len(ifaces) == 0 { + message("No interfaces found for validating IP configuration") + } + + for _, i := range ifaces { + for _, c := range i.Cidrs { + if ip.Equal(c.IP) { + message("IPv%d address %s discovered on interface %s", ip.Version(), ip.String(), i.Name) + return + } + } + } + warning("Unable to confirm IPv%d address %s is assigned to this host", ip.Version(), ip) +} + +// autoDetectCIDR auto-detects the IP and Network using the requested +// detection method. +func autoDetectCIDR(method string, version int) *net.IPNet { + if method == "" || method == AUTODETECTION_METHOD_FIRST { + // Autodetect the IP by enumerating all interfaces (excluding + // known internal interfaces). + return autoDetectCIDRFirstFound(version) + } else if strings.HasPrefix(method, AUTODETECTION_METHOD_INTERFACE) { + // Autodetect the IP from the specified interface. + ifStr := strings.TrimPrefix(method, AUTODETECTION_METHOD_INTERFACE) + return autoDetectCIDRByInterface(ifStr, version) + } else if strings.HasPrefix(method, AUTODETECTION_METHOD_CAN_REACH) { + // Autodetect the IP by connecting a UDP socket to a supplied address. + destStr := strings.TrimPrefix(method, AUTODETECTION_METHOD_CAN_REACH) + return autoDetectCIDRByReach(destStr, version) + } + + // The autodetection method is not recognised and is required. Exit. + fatal("Invalid IP autodection method: %s", method) + terminate() + return nil +} + +// autoDetectCIDRFirstFound auto-detects the first valid Network it finds across +// all interfaces (excluding common known internal interface names). +func autoDetectCIDRFirstFound(version int) *net.IPNet { + incl := []string{} + excl := []string{ + "docker.*", "cbr.*", "dummy.*", + "virbr.*", "lxcbr.*", "veth.*", "lo", + "cali.*", "tunl.*", "flannel.*", + } + + iface, cidr, err := autodetection.FilteredEnumeration(incl, excl, version) + if err != nil { + warning("Unable to auto-detect an IPv%d address: %s", version, err) + return nil + } + + message("Using autodetected IPv%d address on interface %s: %s", version, iface.Name, cidr.String()) + + return cidr +} + +// autoDetectCIDRByInterface auto-detects the first valid Network on the interfaces +// matching the supplied interface regex. +func autoDetectCIDRByInterface(ifaceRegex string, version int) *net.IPNet { + iface, cidr, err := autodetection.FilteredEnumeration([]string{ifaceRegex}, nil, version) + if err != nil { + warning("Unable to auto-detect an IPv%d address using interface regex %s: %s", version, ifaceRegex, err) + return nil + } + + message("Using autodetected IPv%d address %s on matching interface %s", version, cidr.String(), iface.Name) + + return cidr +} + +// autoDetectCIDRByReach auto-detects the IP and Network by setting up a UDP +// connection to a "reach" address. +func autoDetectCIDRByReach(dest string, version int) *net.IPNet { + if cidr, err := autodetection.ReachDestination(dest, version); err != nil { + warning("Unable to auto-detect IPv%d address by connecting to %s: %s", version, dest, err) + return nil + } else { + message("Using autodetected IPv%d address %s, detected by connecting to %s", version, cidr.String(), dest) + return cidr + } +} + +// configureASNumber configures the Node resource with the AS number specified +// in the environment, or is a no-op if not specified. +func configureASNumber(node *api.Node) { + // Extract the AS number from the environment + asStr := os.Getenv("AS") + if asStr != "" { + if asNum, err := numorstring.ASNumberFromString(asStr); err != nil { + fatal("The AS number specified in the environment (AS=%s) is not valid: %s", asStr, err) + terminate() + } else { + message("Using AS number specified in environment (AS=%s)", asNum) + node.Spec.BGP.ASNumber = &asNum + } + } else { + if node.Spec.BGP.ASNumber == nil { + message("No AS number configured on node resource, using global value") + } else { + message("Using AS number %s configured in node resource", node.Spec.BGP.ASNumber) + } + } +} + +// configureIPPools ensures that default IP pools are created (unless explicitly +// requested otherwise). +func configureIPPools(client *client.Client) { + // Read in environment variables for use here and later. + ipv4Pool := os.Getenv("CALICO_IPV4POOL_CIDR") + ipv6Pool := os.Getenv("CALICO_IPV6POOL_CIDR") + + if strings.ToLower(os.Getenv("NO_DEFAULT_POOLS")) == "true" { + if len(ipv4Pool) > 0 || len(ipv6Pool) > 0 { + fatal("Invalid configuration with NO_DEFAULT_POOLS defined and CALICO_IPV4POOL_CIDR or CALICO_IPV6POOL_CIDR defined.") + terminate() + } + + log.Info("Skipping IP pool configuration") + return + } + + ipv4IpipModeEnvVar := strings.ToLower(os.Getenv("CALICO_IPV4POOL_IPIP")) + + // Get a list of all IP Pools + poolList, err := client.IPPools().List(api.IPPoolMetadata{}) + if err != nil { + fatal("Unable to fetch IP pool list: %s", err) + terminate() + return // not really needed but allows testing to function + } + + // Check for IPv4 and IPv6 pools. + ipv4Present := false + ipv6Present := false + for _, p := range poolList.Items { + version := p.Metadata.CIDR.Version() + ipv4Present = ipv4Present || (version == 4) + ipv6Present = ipv6Present || (version == 6) + if ipv4Present && ipv6Present { + break + } + } + + // Read IPV4 CIDR from env if set and parse then check it for errors + if ipv4Pool == "" { + ipv4Pool = DEFAULT_IPV4_POOL_CIDR + } + _, ipv4Cidr, err := net.ParseCIDR(ipv4Pool) + if err != nil || ipv4Cidr.Version() != 4 { + fatal("Invalid CIDR specified in CALICO_IPV4POOL_CIDR '%s'", ipv4Pool) + terminate() + return // not really needed but allows testing to function + } + + // Read IPV6 CIDR from env if set and parse then check it for errors + if ipv6Pool == "" { + ipv6Pool = DEFAULT_IPV6_POOL_CIDR + } + _, ipv6Cidr, err := net.ParseCIDR(ipv6Pool) + if err != nil || ipv6Cidr.Version() != 6 { + fatal("Invalid CIDR specified in CALICO_IPV6POOL_CIDR '%s'", ipv6Pool) + terminate() + return // not really needed but allows testing to function + } + + // Ensure there are pools created for each IP version. + if !ipv4Present { + log.Debug("Create default IPv4 IP pool") + createIPPool(client, ipv4Cidr, ipv4IpipModeEnvVar) + } + if !ipv6Present && ipv6Supported() { + log.Debug("Create default IPv6 IP pool") + createIPPool(client, ipv6Cidr, string(ipip.Undefined)) + } +} + +// ipv6Supported returns true if IPv6 is supported on this platform. This performs +// a check on the appropriate Felix parameter and if supported also performs a +// simplistic check of /proc/sys/net/ipv6 (since platforms that do not have IPv6 +// compiled in will not have this entry). +func ipv6Supported() bool { + // First check the Felix parm. + switch strings.ToLower(os.Getenv("FELIX_IPV6SUPPORT")) { + case "false", "0", "no", "n", "f": + log.Info("IPv6 support disabled through environment") + return false + } + + // If supported, then also check /proc/sys/net/ipv6. + _, err := os.Stat("/proc/sys/net/ipv6") + supported := (err == nil) + log.Infof("IPv6 supported on this platform: %v", supported) + return supported +} + +// createIPPool creates an IP pool using the specified CIDR. This +// method is a no-op if the pool already exists. +func createIPPool(client *client.Client, cidr *net.IPNet, ipipModeName string) { + version := cidr.Version() + ipipMode := ipip.Mode(ipipModeName) + + // off is not an actual valid value so switch it to an empty string + if ipipModeName == "off" { + ipipMode = ipip.Undefined + } + + pool := &api.IPPool{ + Metadata: api.IPPoolMetadata{ + CIDR: *cidr, + }, + Spec: api.IPPoolSpec{ + NATOutgoing: true, + IPIP: &api.IPIPConfiguration{ + Enabled: ipipMode != ipip.Undefined, + Mode: ipipMode, + }, + }, + } + + // Use off when logging for disabled instead of blank + if ipipMode == ipip.Undefined { + ipipModeName = "off" + } + + log.Infof("Ensure default IPv%d pool is created. IPIP mode: %s", version, ipipModeName) + + // Create the pool. There is a small chance that another node may + // beat us to it, so handle the fact that the pool already exists. + if _, err := client.IPPools().Create(pool); err != nil { + if _, ok := err.(errors.ErrorResourceAlreadyExists); !ok { + fatal("Failed to create default IPv%d IP pool: %s", version, err) + terminate() + } + } else { + message("Created default IPv%d pool (%s) with NAT outgoing enabled. IPIP mode: %s", + version, cidr, ipipModeName) + } +} + +// checkConflictingNodes checks whether any other nodes have been configured +// with the same IP addresses. +func checkConflictingNodes(client *client.Client, node *api.Node) { + // Get the full set of nodes. + var nodes []api.Node + if nodeList, err := client.Nodes().List(api.NodeMetadata{}); err != nil { + fatal("Unable to query node confguration: %s", err) + terminate() + } else { + nodes = nodeList.Items + } + + ourIPv4 := node.Spec.BGP.IPv4Address + ourIPv6 := node.Spec.BGP.IPv6Address + errored := false + for _, theirNode := range nodes { + if theirNode.Spec.BGP == nil { + // Skip nodes that don't have BGP configured. We know + // that this node does have BGP since we only perform + // this check after configuring BGP. + continue + } + theirIPv4 := theirNode.Spec.BGP.IPv4Address + theirIPv6 := theirNode.Spec.BGP.IPv6Address + + // If this is our node (based on the name), check if the IP + // addresses have changed. If so warn the user as it could be + // an indication of multiple nodes using the same name. This + // is not an error condition as the IPs could actually change. + if theirNode.Metadata.Name == node.Metadata.Name { + if theirIPv4 != nil && ourIPv4 != nil && !theirIPv4.IP.Equal(ourIPv4.IP) { + warning("Calico node '%s' IPv4 address has changed:", + theirNode.Metadata.Name) + message(" - This could happen if multiple nodes are configured with the same name") + message(" - Original IP: %s", theirIPv4.IP) + message(" - Updated IP: %s", ourIPv4.IP) + } + if theirIPv6 != nil && ourIPv6 != nil && !theirIPv6.IP.Equal(ourIPv6.IP) { + warning("Calico node '%s' IPv6 address has changed:", + theirNode.Metadata.Name) + message(" - This could happen if multiple nodes are configured with the same name") + message(" - Original IP: %s", theirIPv6.IP) + message(" - Updated IP: %s", ourIPv6.IP) + } + continue + } + + // Check that other nodes aren't using the same IP addresses. + // This is an error condition. + if theirIPv4 != nil && ourIPv4 != nil && theirIPv4.IP.Equal(ourIPv4.IP) { + message("Calico node '%s' is already using the IPv4 address %s:", + theirNode.Metadata.Name, ourIPv4.IP) + message(" - Check the node configuration to remove the IP address conflict") + errored = true + } + if theirIPv6 != nil && ourIPv6 != nil && theirIPv6.IP.Equal(ourIPv6.IP) { + message("Calico node '%s' is already using the IPv6 address %s:", + theirNode.Metadata.Name, ourIPv6.IP) + message(" - Check the node configuration to remove the IP address conflict") + errored = true + } + } + + if errored { + terminate() + } +} + +// Checks that the filesystem is as expected and fix it if possible +func ensureFilesystemAsExpected() { + // BIRD requires the /var/run/calico directory in order to provide status + // information over the control socket, but other backends do not + // need this check. + if strings.ToLower(os.Getenv("CALICO_NETWORKING_BACKEND")) == "bird" { + runDir := "/var/run/calico" + // Check if directory already exists + if _, err := os.Stat(runDir); err != nil { + // Create the runDir + if err = os.MkdirAll(runDir, os.ModeDir); err != nil { + fatal("Unable to create '%s'", runDir) + terminate() + } + warning("%s was not mounted, 'calicoctl node status' may provide incomplete status information", runDir) + } + } + + // Ensure the log directory exists but only if logging to file is enabled. + if strings.ToLower(os.Getenv("CALICO_DISABLE_FILE_LOGGING")) != "true" { + logDir := "/var/log/calico" + // Check if directory already exists + if _, err := os.Stat(logDir); err != nil { + // Create the logDir + if err = os.MkdirAll(logDir, os.ModeDir); err != nil { + fatal("Unable to create '%s'", logDir) + terminate() + } + warning("%s was not mounted, 'calicoctl node diags' will not be able to collect logs", logDir) + } + } + +} + +// ensureDefaultConfig ensures all of the required default settings are +// configured. +func ensureDefaultConfig(c *client.Client, node *api.Node) error { + // By default we set the global reporting interval to 0 - this is + // different from the defaults defined in Felix. + // + // Logging to file is disabled in the felix.cfg config file. This + // should always be disabled for calico/node. By default we log to + // screen - set the default logging value that we desire. + if err := ensureGlobalFelixConfig(c, "ReportingIntervalSecs", "0"); err != nil { + return err + } else if err = ensureGlobalFelixConfig(c, "LogSeverityScreen", client.GlobalDefaultLogLevel); err != nil { + return err + } + + // Configure Felix to allow traffic from the containers to the host (if + // not otherwise firewalled by the host administrator or profiles). + // This is important for container deployments, where it is common + // for containers to speak to services running on the host (e.g. k8s + // pods speaking to k8s api-server, and mesos tasks registering with agent + // on startup). + if err := ensureFelixConfig(c, node.Metadata.Name, "DefaultEndpointToHostAction", "RETURN"); err != nil { + return err + } + + // Set the default values for some of the global BGP config values and + // per-node directory structure. + // These are required by both confd and the GoBGP daemon. Some of this + // can only be done directly by the backend (since it requires access to + // datastore features not exposed in the main API). + // + // TODO: This is only required for the current BIRD and GoBGP integrations, + // but should be removed once we switch over to a better watcher interface. + if err := ensureGlobalBGPConfig(c, "node_mesh", fmt.Sprintf("{\"enabled\": %v}", client.GlobalDefaultNodeToNodeMesh)); err != nil { + return err + } else if err := ensureGlobalBGPConfig(c, "as_num", strconv.Itoa(client.GlobalDefaultASNumber)); err != nil { + return err + } else if err = ensureGlobalBGPConfig(c, "loglevel", client.GlobalDefaultLogLevel); err != nil { + return err + } else if err = c.Backend.EnsureCalicoNodeInitialized(node.Metadata.Name); err != nil { + return err + } + return nil +} + +// ensureGlobalFelixConfig ensures that the supplied global felix config value +// is initialized, and if not initialize it with the supplied default. +func ensureGlobalFelixConfig(c *client.Client, key, def string) error { + if val, assigned, err := c.Config().GetFelixConfig(key, ""); err != nil { + return err + } else if !assigned { + return c.Config().SetFelixConfig(key, "", def) + } else { + log.WithField(key, val).Debug("Global Felix value already assigned") + return nil + } +} + +// ensureFelixConfig ensures that the supplied felix config value +// is initialized, and if not initialize it with the supplied default. +func ensureFelixConfig(c *client.Client, host, key, def string) error { + if val, assigned, err := c.Config().GetFelixConfig(key, host); err != nil { + return err + } else if !assigned { + return c.Config().SetFelixConfig(key, host, def) + } else { + log.WithField(key, val).Debug("Host Felix value already assigned") + return nil + } +} + +// ensureGlobalBGPConfig ensures that the supplied global BGP config value +// is initialized, and if not initialize it with the supplied default. +func ensureGlobalBGPConfig(c *client.Client, key, def string) error { + if val, assigned, err := c.Config().GetBGPConfig(key, ""); err != nil { + return err + } else if !assigned { + return c.Config().SetBGPConfig(key, "", def) + } else { + log.WithField(key, val).Debug("Global BGP value already assigned") + return nil + } +} + +// message prints a message to screen and to log. A newline terminator is +// not required in the format string. +func message(format string, args ...interface{}) { + fmt.Printf(format+"\n", args...) +} + +// warning prints a warning to screen and to log. A newline terminator is +// not required in the format string. +func warning(format string, args ...interface{}) { + fmt.Printf("WARNING: "+format+"\n", args...) +} + +// fatal prints a fatal message to screen and to log. A newline terminator is +// not required in the format string. +func fatal(format string, args ...interface{}) { + fmt.Printf("ERROR: "+format+"\n", args...) +} + +// terminate prints a terminate message and exists with status 1. +func terminate() { + message("Terminating") + exitFunction(1) +} diff --git a/startup/startup_suite_test.go b/startup/startup_suite_test.go new file mode 100644 index 000000000..df1602d33 --- /dev/null +++ b/startup/startup_suite_test.go @@ -0,0 +1,19 @@ +package main_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" + + "github.com/projectcalico/libcalico-go/lib/testutils" +) + +func init() { + testutils.HookLogrusForGinkgo() +} + +func TestCommands(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Startup Suite") +} diff --git a/startup/startup_test.go b/startup/startup_test.go new file mode 100644 index 000000000..88194abde --- /dev/null +++ b/startup/startup_test.go @@ -0,0 +1,268 @@ +// Copyright (c) 2016 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + "os" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + "github.com/projectcalico/libcalico-go/lib/api" + "github.com/projectcalico/libcalico-go/lib/client" + "github.com/projectcalico/libcalico-go/lib/ipip" + "github.com/projectcalico/libcalico-go/lib/testutils" +) + +var exitCode int + +func fakeExitFunction(ec int) { + exitCode = ec +} + +var _ = Describe("Non-etcd related tests", func() { + + Describe("Logging tests", func() { + Context("Test message", func() { + message("Test message %d, %s", 4, "END") + }) + Context("Test warning", func() { + warning("Test message %d, %s", 4, "END") + }) + Context("Test fatal", func() { + fatal("Test message %d, %s", 4, "END") + }) + }) + + Describe("Termination tests", func() { + exitCode = 0 + Context("Test termination", func() { + oldExit := exitFunction + exitFunction = fakeExitFunction + defer func() { exitFunction = oldExit }() + terminate() + It("should have terminated", func() { + Expect(exitCode).To(Equal(1)) + }) + }) + }) +}) + +type EnvItem struct { + key string + value string +} + +var _ = Describe("FV tests against a real etcd", func() { + changedEnvVars := []string{"CALICO_IPV4POOL_CIDR", "CALICO_IPV6POOL_CIDR", "NO_DEFAULT_POOLS", "CALICO_IPV4POOL_IPIP"} + + BeforeEach(func() { + for _, envName := range changedEnvVars { + os.Unsetenv(envName) + } + }) + AfterEach(func() { + for _, envName := range changedEnvVars { + os.Unsetenv(envName) + } + }) + + DescribeTable("Test IP pool env variables", + func(envList []EnvItem, expectedIPv4 string, expectedIPv6 string, expectIpv4IpipMode string) { + // Create a new client. + cfg, _ := client.LoadClientConfigFromEnvironment() + c := testutils.CreateCleanClient(*cfg) + + // Set the env variables specified. + for _, env := range envList { + os.Setenv(env.key, env.value) + } + poolList, err := c.IPPools().List(api.IPPoolMetadata{}) + Expect(poolList.Items).To(BeEmpty()) + + // Run the UUT. + configureIPPools(c) + + // Get the IPPool list. + poolList, err = c.IPPools().List(api.IPPoolMetadata{}) + Expect(err).NotTo(HaveOccurred()) + log.Println("Get pool list returns: ", poolList.Items) + + // Look through the pool for the expected data. + foundv4Expected := false + foundv6Expected := false + for _, pool := range poolList.Items { + if pool.Metadata.CIDR.String() == expectedIPv4 { + foundv4Expected = true + } + if pool.Metadata.CIDR.String() == expectedIPv6 { + foundv6Expected = true + } + if pool.Metadata.CIDR.Version() == 6 { + // Expect IPIP on IPv6 to be disabled + if pool.Spec.IPIP != nil { + Expect(pool.Spec.IPIP.Enabled).To(BeFalse()) + } + } else { + // off is not a real mode value but use it instead of empty string + if expectIpv4IpipMode == "off" { + if pool.Spec.IPIP != nil { + Expect(pool.Spec.IPIP.Enabled).To(BeFalse()) + } + } else { + Expect(pool.Spec.IPIP.Enabled).To(BeTrue()) + Expect(pool.Spec.IPIP.Mode).To(Equal(ipip.Mode(expectIpv4IpipMode))) + } + } + } + Expect(foundv4Expected).To(BeTrue(), + "Expected %s to be in Pools", expectedIPv4) + Expect(foundv6Expected).To(BeTrue(), + "Expected %s to be in Pools", expectedIPv6) + }, + + Entry("No env variables set", []EnvItem{}, + "192.168.0.0/16", "fd80:24e2:f998:72d6::/64", "off"), + Entry("IPv4 Pool env var set", + []EnvItem{{"CALICO_IPV4POOL_CIDR", "172.16.0.0/24"}}, + "172.16.0.0/24", "fd80:24e2:f998:72d6::/64", "off"), + Entry("IPv6 Pool env var set", + []EnvItem{{"CALICO_IPV6POOL_CIDR", "fdff:ffff:ffff:ffff:ffff::/80"}}, + "192.168.0.0/16", "fdff:ffff:ffff:ffff:ffff::/80", "off"), + Entry("Both IPv4 and IPv6 Pool env var set", + []EnvItem{ + {"CALICO_IPV4POOL_CIDR", "172.16.0.0/24"}, + {"CALICO_IPV6POOL_CIDR", "fdff:ffff:ffff:ffff:ffff::/80"}, + }, + "172.16.0.0/24", "fdff:ffff:ffff:ffff:ffff::/80", "off"), + Entry("CALICO_IPV4POOL_IPIP set off", []EnvItem{{"CALICO_IPV4POOL_IPIP", "off"}}, + "192.168.0.0/16", "fd80:24e2:f998:72d6::/64", "off"), + Entry("CALICO_IPV4POOL_IPIP set always", []EnvItem{{"CALICO_IPV4POOL_IPIP", "always"}}, + "192.168.0.0/16", "fd80:24e2:f998:72d6::/64", "always"), + Entry("CALICO_IPV4POOL_IPIP set cross-subnet", []EnvItem{{"CALICO_IPV4POOL_IPIP", "cross-subnet"}}, + "192.168.0.0/16", "fd80:24e2:f998:72d6::/64", "cross-subnet"), + Entry("IPv6 Pool and IPIP set", + []EnvItem{ + {"CALICO_IPV6POOL_CIDR", "fdff:ffff:ffff:ffff:ffff::/80"}, + {"CALICO_IPV4POOL_IPIP", "always"}, + }, + "192.168.0.0/16", "fdff:ffff:ffff:ffff:ffff::/80", "always"), + ) + + Describe("Test NO_DEFAULT_POOLS env variable", func() { + Context("Should have no pools defined", func() { + // Create a new client. + cfg, _ := client.LoadClientConfigFromEnvironment() + c := testutils.CreateCleanClient(*cfg) + + // Set the env variables specified. + os.Setenv("NO_DEFAULT_POOLS", "true") + + // Run the UUT. + configureIPPools(c) + + // Get the IPPool list. + poolList, err := c.IPPools().List(api.IPPoolMetadata{}) + It("should be able to access the IP pool list", func() { + Expect(err).NotTo(HaveOccurred()) + }) + log.Println("Get pool list returns: ", poolList.Items) + + It("should have no IP pools", func() { + Expect(poolList.Items).To(BeEmpty(), "Environment %#v", os.Environ()) + }) + }) + }) + + DescribeTable("Test IP pool env variables that cause exit", + func(envList []EnvItem) { + my_ec := 0 + oldExit := exitFunction + exitFunction = func(ec int) { my_ec = ec } + defer func() { exitFunction = oldExit }() + + // Create a new client. + cfg, _ := client.LoadClientConfigFromEnvironment() + c := testutils.CreateCleanClient(*cfg) + + // Set the env variables specified. + for _, env := range envList { + os.Setenv(env.key, env.value) + } + + // Run the UUT. + configureIPPools(c) + + Expect(my_ec).To(Equal(1)) + }, + + Entry("Bad IPv4 Pool CIDR", []EnvItem{{"CALICO_IPV4POOL_CIDR", "172.16.0.0a/24"}}), + Entry("Too small IPv4 Pool CIDR", []EnvItem{{"CALICO_IPV4POOL_CIDR", "172.16.0.0/27"}}), + Entry("Single IPv4 is too small for a pool CIDR", []EnvItem{{"CALICO_IPV4POOL_CIDR", "10.0.0.0/32"}}), + Entry("Small IPv6 is too small for a pool CIDR", []EnvItem{{"CALICO_IPV6POOL_CIDR", "fd00::/123"}}), + Entry("Bad IPv4 Pool with good IPv6 Pool env var set", + []EnvItem{ + {"CALICO_IPV4POOL_CIDR", "172.16.0.0a/24"}, + {"CALICO_IPV6POOL_CIDR", "fdff:ffff:ffff:ffff:ffff::/80"}, + }), + Entry("Invalid Env Var combo", + []EnvItem{ + {"NO_DEFAULT_POOLS", "true"}, + {"CALICO_IPV4POOL_CIDR", "172.16.0.0/24"}, + }), + Entry("Bad IPv4 Pool IPIP Mode", []EnvItem{{"CALICO_IPV4POOL_IPIP", "badVal"}}), + Entry("v6 Address in IPv4 Pool CIDR", + []EnvItem{{"CALICO_IPV4POOL_CIDR", "fdff:ffff:ffff:ffff:ffff::/80"}}), + Entry("v4 Address in IPv6 Pool CIDR", + []EnvItem{{"CALICO_IPV6POOL_CIDR", "172.16.0.0/24"}}), + ) + + Describe("Test we properly wait for the etcd datastore", func() { + // Create a new client. + cfg, _ := client.LoadClientConfigFromEnvironment() + c := testutils.CreateCleanClient(*cfg) + + // Wait for a connection. + done := make(chan bool) + go func() { + // Wait for a connection. + waitForConnection(c) + + // Once connected, indicate that we connected on the channel. + done <- true + }() + + // Wait for a done signal to indicate that we've connected to the datastore. + // If we don't receive one in 5 seconds, then fail. + count := 0 + for { + select { + case <-done: + // Finished. Success! + log.Println("Connected to datastore") + return + default: + count++ + time.Sleep(1 * time.Second) + if count > 5 { + log.Fatal("Timed out waiting for datastore after 5 seconds") + } + } + } + }) +}) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/__init__.py b/tests/st/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/bgp/__init__.py b/tests/st/bgp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/bgp/peer.py b/tests/st/bgp/peer.py new file mode 100644 index 000000000..b1c4fa4a7 --- /dev/null +++ b/tests/st/bgp/peer.py @@ -0,0 +1,31 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def create_bgp_peer(host, scope, ip, asNum): + assert scope in ('node', 'global') + node = host.get_hostname() if scope == 'node' else "" + testdata = { + 'apiVersion': 'v1', + 'kind': 'bgpPeer', + 'metadata': { + 'scope': scope, + 'node': node, + 'peerIP': ip, + }, + 'spec': { + 'asNumber': asNum + } + } + host.writefile("testfile.yaml", testdata) + host.calicoctl("create -f testfile.yaml") diff --git a/tests/st/bgp/test_backends.py b/tests/st/bgp/test_backends.py new file mode 100644 index 000000000..a03dc3e76 --- /dev/null +++ b/tests/st/bgp/test_backends.py @@ -0,0 +1,73 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.constants import (DEFAULT_IPV4_ADDR_1, DEFAULT_IPV4_ADDR_2, + DEFAULT_IPV4_ADDR_3, + DEFAULT_IPV4_POOL_CIDR, LARGE_AS_NUM) +from tests.st.utils.utils import check_bird_status + +class TestBGPBackends(TestBase): + + @attr('slow') + def test_bgp_backends(self): + """ + Test using different BGP backends. + + We run a multi-host test for this to test peering between two gobgp + backends and a single BIRD backend. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2, \ + DockerHost('host3', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=True) as host3: + + # Set the default AS number. + host1.calicoctl("config set asNumber %s" % LARGE_AS_NUM) + + # Start host1 using the inherited AS, and host2 using a specified + # AS (same as default). These hosts use the gobgp backend, whereas + # host3 uses BIRD. + host1.start_calico_node("--backend=gobgp") + host2.start_calico_node("--backend=gobgp --as=%s" % LARGE_AS_NUM) + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1", subnet=DEFAULT_IPV4_POOL_CIDR) + workload_host1 = host1.create_workload("workload1", network=network1, ip=DEFAULT_IPV4_ADDR_1) + workload_host2 = host2.create_workload("workload2", network=network1, ip=DEFAULT_IPV4_ADDR_2) + workload_host3 = host3.create_workload("workload3", network=network1, ip=DEFAULT_IPV4_ADDR_3) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2, + workload_host3], + ip_pass_list=[workload_host1.ip, + workload_host2.ip, + workload_host3.ip]) + + # Check the BGP status on the BIRD/GoBGP host. + hosts = [host1, host2, host3] + for target in hosts: + expected = [("node-to-node mesh", h.ip, "Established") for h in hosts if h is not target] + check_bird_status(target, expected) diff --git a/tests/st/bgp/test_global_config.py b/tests/st/bgp/test_global_config.py new file mode 100644 index 000000000..4d999fb2e --- /dev/null +++ b/tests/st/bgp/test_global_config.py @@ -0,0 +1,92 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.constants import (DEFAULT_IPV4_ADDR_1, DEFAULT_IPV4_ADDR_2, + DEFAULT_IPV4_POOL_CIDR, LARGE_AS_NUM) +from tests.st.utils.exceptions import CommandExecError +from tests.st.utils.utils import check_bird_status + +class TestBGP(TestBase): + + def test_defaults(self): + """ + Test default BGP configuration commands. + """ + with DockerHost('host', start_calico=False, dind=False) as host: + # Check default AS command + self.assertEquals(host.calicoctl("config get asNumber"), "64512") + host.calicoctl("config set asNumber 12345") + self.assertEquals(host.calicoctl("config get asNumber"), "12345") + with self.assertRaises(CommandExecError): + host.calicoctl("config set asNumber 99999999999999999999999") + with self.assertRaises(CommandExecError): + host.calicoctl("config set asNumber abcde") + + # Check BGP mesh command + self.assertEquals(host.calicoctl("config get nodeToNodeMesh"), "on") + host.calicoctl("config set nodeToNodeMesh off") + self.assertEquals(host.calicoctl("config get nodeToNodeMesh"), "off") + host.calicoctl("config set nodeToNodeMesh on") + self.assertEquals(host.calicoctl("config get nodeToNodeMesh"), "on") + + @attr('slow') + def _test_as_num(self, backend='bird'): + """ + Test using different AS number for the node-to-node mesh. + + We run a multi-host test for this as we need to set up real BGP peers. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2: + + # Set the default AS number. + host1.calicoctl("config set asNumber %s" % LARGE_AS_NUM) + + # Start host1 using the inherited AS, and host2 using a specified + # AS (same as default). + host1.start_calico_node("--backend=%s" % backend) + host2.start_calico_node("--backend=%s --as=%s" % (backend, LARGE_AS_NUM)) + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1", subnet=DEFAULT_IPV4_POOL_CIDR) + workload_host1 = host1.create_workload("workload1", network=network1, ip=DEFAULT_IPV4_ADDR_1) + workload_host2 = host2.create_workload("workload2", network=network1, ip=DEFAULT_IPV4_ADDR_2) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[DEFAULT_IPV4_ADDR_1, + DEFAULT_IPV4_ADDR_2]) + + # Check the BGP status on each host. + check_bird_status(host1, [("node-to-node mesh", host2.ip, "Established")]) + check_bird_status(host2, [("node-to-node mesh", host1.ip, "Established")]) + + @attr('slow') + def test_bird_as_num(self): + self._test_as_num(backend='bird') + + @attr('slow') + def test_gobgp_as_num(self): + self._test_as_num(backend='gobgp') diff --git a/tests/st/bgp/test_global_peers.py b/tests/st/bgp/test_global_peers.py new file mode 100644 index 000000000..6057dfe26 --- /dev/null +++ b/tests/st/bgp/test_global_peers.py @@ -0,0 +1,84 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.constants import (DEFAULT_IPV4_ADDR_1, DEFAULT_IPV4_ADDR_2, + DEFAULT_IPV4_POOL_CIDR, LARGE_AS_NUM) +from tests.st.utils.utils import check_bird_status + +from .peer import create_bgp_peer + +class TestGlobalPeers(TestBase): + + def _test_global_peers(self, backend='bird'): + """ + Test global BGP peer configuration. + + Test by turning off the mesh and configuring the mesh as + a set of global peers. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2: + # Start both hosts using specific AS numbers. + host1.start_calico_node("--backend=%s --as=%s" % (backend, LARGE_AS_NUM)) + host2.start_calico_node("--backend=%s --as=%s" % (backend, LARGE_AS_NUM)) + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1", subnet=DEFAULT_IPV4_POOL_CIDR) + workload_host1 = host1.create_workload("workload1", network=network1, ip=DEFAULT_IPV4_ADDR_1) + workload_host2 = host2.create_workload("workload2", network=network1, ip=DEFAULT_IPV4_ADDR_2) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Turn the node-to-node mesh off and wait for connectivity to drop. + host1.calicoctl("config set nodeToNodeMesh off") + self.assert_true(workload_host1.check_cant_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Configure global peers to explicitly set up a mesh. This means + # each node will try to peer with itself which will fail. + create_bgp_peer(host1, 'global', host2.ip, LARGE_AS_NUM) + create_bgp_peer(host2, 'global', host1.ip, LARGE_AS_NUM) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[DEFAULT_IPV4_ADDR_1, + DEFAULT_IPV4_ADDR_2]) + + # Check the BGP status on each host. Connections from a node to + # itself will be idle since this is invalid BGP configuration. + check_bird_status(host1, [("global", host1.ip, ["Idle", "Active"]), + ("global", host2.ip, "Established")]) + check_bird_status(host2, [("global", host1.ip, "Established"), + ("global", host2.ip, ["Idle", "Active"])]) + + @attr('slow') + def test_bird_node_peers(self): + self._test_global_peers(backend='bird') + + @attr('slow') + def test_gobgp_node_peers(self): + self._test_global_peers(backend='gobgp') diff --git a/tests/st/bgp/test_ipip.py b/tests/st/bgp/test_ipip.py new file mode 100644 index 000000000..cb91d3ab2 --- /dev/null +++ b/tests/st/bgp/test_ipip.py @@ -0,0 +1,419 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import re +import subprocess + +from netaddr import IPAddress, IPNetwork +from nose_parameterized import parameterized +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.constants import DEFAULT_IPV4_POOL_CIDR +from tests.st.utils.route_reflector import RouteReflectorCluster +from tests.st.utils.utils import check_bird_status, retry_until_success +from time import sleep + +from .peer import create_bgp_peer + +""" +Test calico IPIP behaviour. +""" + +class TestIPIP(TestBase): + def tearDown(self): + self.remove_tunl_ip() + + @parameterized.expand([ + ('bird',), + ('gobgp',), + ]) + def test_ipip(self, backend): + """ + Test IPIP routing with the different IPIP modes. + + This test modifies the working IPIP mode of the pool and monitors the + traffic flow to ensure it either is or is not going over the IPIP + tunnel as expected. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2: + + # Before starting the node, create the default IP pool using the + # v1.0.2 calicoctl. For calicoctl v1.1.0+, a new IPIP mode field + # is introduced - by testing with an older pool version validates + # the IPAM BIRD templates function correctly without the mode field. + self.pool_action(host1, "create", DEFAULT_IPV4_POOL_CIDR, False, + calicoctl_version="v1.0.2") + + # Autodetect the IP addresses - this should ensure the subnet is + # correctly configured. + host1.start_calico_node("--ip=autodetect --backend={0}".format(backend)) + host2.start_calico_node("--ip=autodetect --backend={0}".format(backend)) + + # Create a network and a workload on each host. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", + network=network1) + workload_host2 = host2.create_workload("workload2", + network=network1) + + # Allow network to converge. + self.assert_true( + workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[workload_host1.ip, + workload_host2.ip]) + + # Note in the following we are making a number of configuration + # changes and testing whether or not IPIP is being used. + # The order of tests is deliberately chosen to flip between IPIP + # and no IPIP because it is easier to look for a change of state + # than to look for state remaining the same. + + # Turn on IPIP with a v1.0.2 calicoctl and check that the + # IPIP tunnel is being used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, True, + calicoctl_version="v1.0.2") + self.assert_ipip_routing(host1, workload_host1, workload_host2, + True) + + # Turn off IPIP using the latest version of calicoctl and check that + # IPIP tunnel is not being used. We'll use the latest version of + # calicoctl for the remaining tests. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, False) + self.assert_ipip_routing(host1, workload_host1, workload_host2, + False) + + # Turn on IPIP, default mode (which is always use IPIP), and check + # IPIP tunnel is being used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, True) + self.assert_ipip_routing(host1, workload_host1, workload_host2, + True) + + # Turn off IPIP and check IPIP tunnel is not being used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, False) + self.assert_ipip_routing(host1, workload_host1, workload_host2, + False) + + # Turn on IPIP mode "always", and check IPIP tunnel is being used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, True, + ipip_mode="always") + self.assert_ipip_routing(host1, workload_host1, workload_host2, + True) + + # Turn on IPIP mode "cross-subnet", since both hosts will be on the + # same subnet, IPIP should not be used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, True, + ipip_mode="cross-subnet") + self.assert_ipip_routing(host1, workload_host1, workload_host2, + False) + + # Set the BGP subnet on both node resources to be a /32. This will + # fool Calico into thinking they are on different subnets. IPIP + # routing should be used. + self.pool_action(host1, "replace", DEFAULT_IPV4_POOL_CIDR, True, + ipip_mode="cross-subnet") + self.modify_subnet(host1, 32) + self.modify_subnet(host2, 32) + self.assert_ipip_routing(host1, workload_host1, workload_host2, + True) + + def test_ipip_addr_assigned(self): + with DockerHost('host', dind=False, start_calico=False) as host: + # Set up first pool before Node is started, to ensure we get tunl IP on boot + ipv4_pool = IPNetwork("10.0.1.0/24") + self.pool_action(host, "create", ipv4_pool, True) + host.start_calico_node() + self.assert_tunl_ip(host, ipv4_pool, expect=True) + + # Disable the IP Pool, and make sure the tunl IP is not from this IP pool anymore. + self.pool_action(host, "apply", ipv4_pool, True, disabled=True) + self.assert_tunl_ip(host, ipv4_pool, expect=False) + + # Re-enable the IP pool and make sure the tunl IP is assigned from that IP pool again. + self.pool_action(host, "apply", ipv4_pool, True) + self.assert_tunl_ip(host, ipv4_pool, expect=True) + + # Test that removing pool removes the tunl IP. + self.pool_action(host, "delete", ipv4_pool, True) + self.assert_tunl_ip(host, ipv4_pool, expect=False) + + # Test that re-adding the pool triggers the confd watch and we get an IP + self.pool_action(host, "create", ipv4_pool, True) + self.assert_tunl_ip(host, ipv4_pool, expect=True) + + # Test that by adding another pool, then deleting the first, + # we remove the original IP, and allocate a new one from the new pool + new_ipv4_pool = IPNetwork("192.168.0.0/16") + self.pool_action(host, "create", new_ipv4_pool, True) + self.pool_action(host, "delete", ipv4_pool, True) + self.assert_tunl_ip(host, new_ipv4_pool) + + def pool_action(self, host, action, cidr, ipip, disabled=False, ipip_mode="", calicoctl_version=None): + """ + Perform an ipPool action. + """ + testdata = { + 'apiVersion': 'v1', + 'kind': 'ipPool', + 'metadata': { + 'cidr': str(cidr) + }, + 'spec': { + 'ipip': { + 'enabled': ipip + }, + 'disabled': disabled + } + } + + # Only add the mode field is a value is specified. Note that + # the mode field will not be valid on pre-v1.1.0 versions of calicoctl. + if ipip_mode: + testdata['spec']['ipip']['mode'] = ipip_mode + + host.writefile("testfile.yaml", testdata) + host.calicoctl("%s -f testfile.yaml" % action, version=calicoctl_version) + + def assert_tunl_ip(self, host, ip_network, expect=True): + """ + Helper function to make assertions on whether or not the tunl interface + on the Host has been assigned an IP or not. This function will retry + 7 times, ensuring that our 5 second confd watch will trigger. + + :param host: DockerHost object + :param ip_network: IPNetwork object which describes the ip-range we do (or do not) + expect to see an IP from on the tunl interface. + :param expect: Whether or not we are expecting to see an IP from IPNetwork on the tunl interface. + :return: + """ + retries = 7 + for retry in range(retries + 1): + try: + output = host.execute("ip addr show tunl0") + match = re.search(r'inet ([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})', output) + if match: + ip_address = IPAddress(match.group(1)) + if expect: + self.assertIn(ip_address, ip_network) + else: + self.assertNotIn(ip_address, ip_network) + else: + self.assertFalse(expect, "No IP address assigned to tunl interface.") + except Exception as e: + if retry < retries: + sleep(1) + else: + raise e + else: + return + + def remove_tunl_ip(self): + """ + Remove the host tunl IP address if assigned. + """ + try: + output = subprocess.check_output(["ip", "addr", "show", "tunl0"]) + except subprocess.CalledProcessError: + return + + match = re.search(r'inet ([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})', output) + if not match: + return + + ipnet = str(IPNetwork(match.group(1))) + + try: + output = subprocess.check_output(["ip", "addr", "del", ipnet, "dev", "tunl0"]) + except subprocess.CalledProcessError: + return + + def modify_subnet(self, host, prefixlen): + """ + Update the calico node resource to use the specified prefix length. + + Returns the current mask size. + """ + node = json.loads(host.calicoctl( + "get node %s --output=json" % host.get_hostname())) + assert len(node) == 1 + + # Get the current network and prefix len + ipnet = IPNetwork(node[0]["spec"]["bgp"]["ipv4Address"]) + current_prefix_len = ipnet.prefixlen + + # Update the prefix length + ipnet.prefixlen = prefixlen + node[0]["spec"]["bgp"]["ipv4Address"] = str(ipnet) + + # Write the data back again. + host.writejson("new_data", node) + host.calicoctl("apply -f new_data") + return current_prefix_len + + def assert_ipip_routing(self, host1, workload_host1, workload_host2, expect_ipip): + """ + Test whether IPIP is being used as expected on host1 when pinging workload_host2 + from workload_host1. + """ + def check(): + orig_tx = self.get_tunl_tx(host1) + workload_host1.execute("ping -c 2 -W 1 %s" % workload_host2.ip) + if expect_ipip: + assert self.get_tunl_tx(host1) == orig_tx + 2 + else: + assert self.get_tunl_tx(host1) == orig_tx + retry_until_success(check) + + def get_tunl_tx(self, host): + """ + Get the tunl TX count + """ + try: + output = host.execute("ifconfig tunl0") + except subprocess.CalledProcessError: + return + + match = re.search(r'RX packets:(\d+) ', + output) + return int(match.group(1)) + + @parameterized.expand([ + (False,), + (True,), + (False,'gobgp',), + (True,'gobgp',), + ]) + def test_gce(self, with_ipip, backend='bird'): + """Test with and without IP-in-IP routing on simulated GCE instances. + + In this test we simulate GCE instance routing, where there is a router + between the instances, and each instance has a /32 address that appears + not to be directly connected to any subnet. With that setup, + connectivity between workloads on different hosts _should_ require + IP-in-IP to be enabled. We test that we do get connectivity _with_ + IP-in-IP, that we don't get connectivity _without_ IP-in-IP, and that + the situation updates dynamically if we toggle IP-in-IP with workloads + already existing. + + Note that this test targets the BGP implementation, to check that it + does IP-in-IP routing correctly, and handles the underlying GCE + routing, and switches dynamically between IP-in-IP and normal routing + as directed by calicoctl. (In the BIRD case, these are all points for + which we've patched the upstream BIRD code.) But naturally it also + involves calicoctl and confd, so it isn't _only_ about the BGP code. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + simulate_gce_routing=True, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + simulate_gce_routing=True, + start_calico=False) as host2: + + self._test_gce_int(with_ipip, backend, host1, host2, False) + + @parameterized.expand([ + (False,), + (True,), + ]) + def test_gce_rr(self, with_ipip): + """As test_gce except with a route reflector instead of mesh config.""" + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + simulate_gce_routing=True, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + simulate_gce_routing=True, + start_calico=False) as host2, \ + RouteReflectorCluster(1, 1) as rrc: + + self._test_gce_int(with_ipip, 'bird', host1, host2, rrc) + + def _test_gce_int(self, with_ipip, backend, host1, host2, rrc): + + host1.start_calico_node("--backend={0}".format(backend)) + host2.start_calico_node("--backend={0}".format(backend)) + + # Before creating any workloads, set the initial IP-in-IP state. + host1.set_ipip_enabled(with_ipip) + + if rrc: + # Set the default AS number - as this is used by the RR mesh, + # and turn off the node-to-node mesh (do this from any host). + host1.calicoctl("config set asNumber 64513") + host1.calicoctl("config set nodeToNodeMesh off") + # Peer from each host to the route reflector. + for host in [host1, host2]: + for rr in rrc.get_redundancy_group(): + create_bgp_peer(host, "node", rr.ip, 64513) + + # Create a network and a workload on each host. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", + network=network1) + workload_host2 = host2.create_workload("workload2", + network=network1) + + for _ in [1, 2]: + # Check we do or don't have connectivity between the workloads, + # according to the IP-in-IP setting. + if with_ipip: + # Allow network to converge. + self.assert_true( + workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[workload_host1.ip, + workload_host2.ip]) + + # Check that we are using IP-in-IP for some routes. + assert "tunl0" in host1.execute("ip r") + assert "tunl0" in host2.execute("ip r") + + # Check that routes are not flapping: the following shell + # script checks that there is no output for 10s from 'ip + # monitor', on either host. The "-le 1" is to allow for + # something (either 'timeout' or 'ip monitor', not sure) saying + # 'Terminated' when the 10s are up. (Note that all commands + # here are Busybox variants; I tried 'grep -v' to eliminate the + # Terminated line, but for some reason it didn't work.) + for host in [host1, host2]: + host.execute("changes=`timeout -t 10 ip -t monitor 2>&1`; " + + "echo \"$changes\"; " + + "test `echo \"$changes\" | wc -l` -le 1") + else: + # Expect non-connectivity between workloads on different hosts. + self.assert_false( + workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + if not rrc: + # Check the BGP status on each host. + check_bird_status(host1, [("node-to-node mesh", host2.ip, "Established")]) + check_bird_status(host2, [("node-to-node mesh", host1.ip, "Established")]) + + # Flip the IP-in-IP state for the next iteration. + with_ipip = not with_ipip + host1.set_ipip_enabled(with_ipip) diff --git a/tests/st/bgp/test_node_peers.py b/tests/st/bgp/test_node_peers.py new file mode 100644 index 000000000..c12209eb1 --- /dev/null +++ b/tests/st/bgp/test_node_peers.py @@ -0,0 +1,81 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.constants import (DEFAULT_IPV4_ADDR_1, DEFAULT_IPV4_ADDR_2, + DEFAULT_IPV4_POOL_CIDR, LARGE_AS_NUM) +from tests.st.utils.utils import check_bird_status + +from .peer import create_bgp_peer + +class TestNodePeers(TestBase): + + def _test_node_peers(self, backend='bird'): + """ + Test per-node BGP peer configuration. + + Test by turning off the mesh and configuring the mesh as + a set of per node peers. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2: + + # Start both hosts using specific AS numbers. + host1.start_calico_node("--backend=%s --as=%s" % (backend, LARGE_AS_NUM)) + host2.start_calico_node("--backend=%s --as=%s" % (backend, LARGE_AS_NUM)) + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1", subnet=DEFAULT_IPV4_POOL_CIDR) + workload_host1 = host1.create_workload("workload1", network=network1, + ip=DEFAULT_IPV4_ADDR_1) + workload_host2 = host2.create_workload("workload2", network=network1, + ip=DEFAULT_IPV4_ADDR_2) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Turn the node-to-node mesh off and wait for connectivity to drop. + host1.calicoctl("config set nodeToNodeMesh off") + self.assert_true(workload_host1.check_cant_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Configure node specific peers to explicitly set up a mesh. + create_bgp_peer(host1, 'node', host2.ip, LARGE_AS_NUM) + create_bgp_peer(host2, 'node', host1.ip, LARGE_AS_NUM) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(DEFAULT_IPV4_ADDR_2, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[DEFAULT_IPV4_ADDR_1, + DEFAULT_IPV4_ADDR_2]) + + # Check the BGP status on each host. + check_bird_status(host1, [("node specific", host2.ip, "Established")]) + check_bird_status(host2, [("node specific", host1.ip, "Established")]) + + @attr('slow') + def test_bird_node_peers(self): + self._test_node_peers(backend='bird') + + @attr('slow') + def test_gobgp_node_peers(self): + self._test_node_peers(backend='gobgp') diff --git a/tests/st/bgp/test_node_status_resilience.py b/tests/st/bgp/test_node_status_resilience.py new file mode 100644 index 000000000..dfe05c0e6 --- /dev/null +++ b/tests/st/bgp/test_node_status_resilience.py @@ -0,0 +1,147 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nose.plugins.attrib import attr +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.constants import (LARGE_AS_NUM) +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.utils import check_bird_status, \ + retry_until_success + +_log = logging.getLogger(__name__) +_log.setLevel(logging.DEBUG) + + +class TestNodeStatusResilience(TestBase): + @parameterized.expand([ + (2, 'bird'), + (0, 'calico-bgp-daemon') + ]) + @attr('slow') + def test_node_status_resilience(self, test_host, pid_name): + """ + Test that newly restarted BGP backend processes consistently + transition to an Established state. + + Test using different BGP backends. + We run a multi-host test for this to test peering between two gobgp + backends and a single BIRD backend. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2, \ + DockerHost('host3', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=True) as host3: + + # Set the default AS number. + host1.calicoctl("config set asNumber %s" % LARGE_AS_NUM) + + # Start host1 using the inherited AS, and host2 using a specified + # AS (same as default). These hosts use the gobgp backend, whereas + # host3 uses BIRD. + host1.start_calico_node("--backend=gobgp") + host2.start_calico_node("--backend=gobgp --as=%s" % LARGE_AS_NUM) + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", network=network1) + workload_host2 = host2.create_workload("workload2", network=network1) + workload_host3 = host3.create_workload("workload3", network=network1) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2, + workload_host3], + ip_pass_list=[workload_host1.ip, + workload_host2.ip, + workload_host3.ip]) + + hosts = [host1, host2, host3] + workloads = [workload_host1, workload_host2, workload_host3] + + _log.debug("==== docker exec -it calico-node ps -a ====") + _log.debug(hosts[test_host].execute("docker exec -it calico-node ps -a")) + + # Check the BGP status on the BIRD/GoBGP host. + def check_connected(): + for target in hosts: + expected = [("node-to-node mesh", h.ip, "Established") for h in hosts if h is not target] + _log.debug("expected : %s", expected) + check_bird_status(target, expected) + + def delete_workload(host, host_workload): + host.calicoctl("ipam release --ip=%s" % host_workload.ip) + host.execute("docker rm -f %s" % host_workload.name) + host.workloads.remove(host_workload) + + def pid_parse(pid_str): + if '\r\n' in pid_str: + pid_list = pid_str.split('\r\n') + return pid_list + else: + return [pid_str] + + iterations = 3 + for iteration in range(1, iterations+1): + _log.debug("Iteration %s", iteration) + _log.debug("Host under test: %s", hosts[test_host].name) + _log.debug("Identify and pkill process: %s", pid_name) + + pre_pkill = hosts[test_host].execute("docker exec -it calico-node pgrep %s" % pid_name) + pre_pkill_list = pid_parse(pre_pkill) + _log.debug("Pre pkill list: %s", pre_pkill_list) + + hosts[test_host].execute("docker exec -it calico-node pkill %s" % pid_name) + + _log.debug('check connected and retry until "Established"') + retry_until_success(check_connected, retries=20, ex_class=Exception) + + post_pkill = hosts[test_host].execute("docker exec -it calico-node pgrep %s" % pid_name) + post_pkill_list = pid_parse(post_pkill) + _log.debug("Post pkill list: %s", post_pkill_list) + + assert pre_pkill_list != post_pkill_list, "The pids should not be the same after pkill" + + new_workloads = [] + for workload in workloads: + new_workload = "%s_%s" % (workload, iteration) + new_workloads.append(new_workload) + + # create new workloads + index = 0 + for new_workload in new_workloads: + new_workload = hosts[index].create_workload(new_workload, network=network1) + _log.debug("host: %s and workload: %s", hosts[index].name, new_workload.name) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2, + workload_host3, + new_workload], + ip_pass_list=[workload_host1.ip, + workload_host2.ip, + workload_host3.ip, + new_workload.ip]) + delete_workload(hosts[index], new_workload) + index += 1 diff --git a/tests/st/bgp/test_route_reflector_cluster.py b/tests/st/bgp/test_route_reflector_cluster.py new file mode 100644 index 000000000..846cda120 --- /dev/null +++ b/tests/st/bgp/test_route_reflector_cluster.py @@ -0,0 +1,86 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.route_reflector import RouteReflectorCluster + +from .peer import create_bgp_peer + +class TestRouteReflectorCluster(TestBase): + + def _test_route_reflector_cluster(self, backend='bird'): + """ + Run a multi-host test using a cluster of route reflectors and node + specific peerings. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2, \ + DockerHost('host3', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host3, \ + RouteReflectorCluster(2, 2) as rrc: + + # Start both hosts using specific backends. + host1.start_calico_node("--backend=%s" % backend) + host2.start_calico_node("--backend=%s" % backend) + host3.start_calico_node("--backend=%s" % backend) + + # Set the default AS number - as this is used by the RR mesh, and + # turn off the node-to-node mesh (do this from any host). + host1.calicoctl("config set asNumber 64513") + host1.calicoctl("config set nodeToNodeMesh off") + + # Create a workload on each host in the same network. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", network=network1) + workload_host2 = host2.create_workload("workload2", network=network1) + workload_host3 = host3.create_workload("workload3", network=network1) + + # Allow network to converge (which it won't) + self.assert_false(workload_host1.check_can_ping(workload_host2.ip, retries=5)) + self.assert_true(workload_host1.check_cant_ping(workload_host3.ip)) + self.assert_true(workload_host2.check_cant_ping(workload_host3.ip)) + + # Set distributed peerings between the hosts, each host peering + # with a different set of redundant route reflectors. + for host in [host1, host2, host3]: + for rr in rrc.get_redundancy_group(): + create_bgp_peer(host, "node", rr.ip, 64513) + + # Allow network to converge (which it now will). + self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=10)) + self.assert_true(workload_host1.check_can_ping(workload_host3.ip, retries=10)) + self.assert_true(workload_host2.check_can_ping(workload_host3.ip, retries=10)) + + # And check connectivity in both directions. + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2, + workload_host3], + ip_pass_list=[workload_host1.ip, + workload_host2.ip, + workload_host3.ip]) + + @attr('slow') + def test_bird_route_reflector_cluster(self): + self._test_route_reflector_cluster(backend='bird') + + @attr('slow') + def test_gobgp_route_reflector_cluster(self): + self._test_route_reflector_cluster(backend='gobgp') diff --git a/tests/st/bgp/test_single_route_reflector.py b/tests/st/bgp/test_single_route_reflector.py new file mode 100644 index 000000000..67ee97320 --- /dev/null +++ b/tests/st/bgp/test_single_route_reflector.py @@ -0,0 +1,78 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.route_reflector import RouteReflectorCluster + +from .peer import create_bgp_peer + +class TestSingleRouteReflector(TestBase): + + @attr('slow') + def _test_single_route_reflector(self, backend='bird'): + """ + Run a multi-host test using a single route reflector and global + peering. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2, \ + RouteReflectorCluster(1, 1) as rrc: + + # Start both hosts using specific backends. + host1.start_calico_node("--backend=%s" % backend) + host2.start_calico_node("--backend=%s" % backend) + + # Set the default AS number - as this is used by the RR mesh, and + # turn off the node-to-node mesh (do this from any host). + host1.calicoctl("config set asNumber 64514") + host1.calicoctl("config set nodeToNodeMesh off") + + # Create a workload on each host in the same network. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", + network=network1) + workload_host2 = host2.create_workload("workload2", + network=network1) + + # Allow network to converge (which it won't) + self.assert_false(workload_host1.check_can_ping(workload_host2.ip, retries=5)) + + # Set global config telling all calico nodes to peer with the + # route reflector. This can be run from either host. + rg = rrc.get_redundancy_group() + assert len(rg) == 1 + create_bgp_peer(host1, "global", rg[0].ip, 64514) + + # Allow network to converge (which it now will). + self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # And check connectivity in both directions. + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[workload_host1.ip, + workload_host2.ip]) + + @attr('slow') + def test_bird_single_route_reflector(self): + self._test_single_route_reflector(backend='bird') + + @attr('slow') + def test_gobgp_single_route_reflector(self): + self._test_single_route_reflector(backend='gobgp') diff --git a/tests/st/bgp/test_update_ip_addr.py b/tests/st/bgp/test_update_ip_addr.py new file mode 100644 index 000000000..fd8540b86 --- /dev/null +++ b/tests/st/bgp/test_update_ip_addr.py @@ -0,0 +1,69 @@ +# Copyright (c) 2017 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS + +class TestUpdateIPAddress(TestBase): + + @attr('slow') + def test_update_ip_address(self): + """ + Test updating the IP address automatically updates and fixes the + Bird BGP config. + """ + with DockerHost('host1', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False) as host2: + + # Start host1 and host2 using bogus IP addresses. The nodes should + # start although they won't be functional. + host1.start_calico_node("--ip=1.2.3.4") + host2.start_calico_node("--ip=2.3.4.5") + + # Create a network and a couple of workloads on each host. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", network=network1) + workload_host2 = host2.create_workload("workload2", network=network1) + + # Fix the node resources to have the correct IP addresses. BIRD + # should automatically fix it's configuration and connectivity will + # be established. + self._fix_ip(host1) + self._fix_ip(host2) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2], + ip_pass_list=[workload_host1.ip, + workload_host2.ip]) + + def _fix_ip(self, host): + """ + Update the calico node resource to have the correct IP for the host. + """ + noder = json.loads(host.calicoctl( + "get node %s --output=json" % host.get_hostname())) + assert len(noder) == 1 + noder[0]["spec"]["bgp"]["ipv4Address"] = str(host.ip) + host.writejson("new_data", noder) + host.calicoctl("apply -f new_data") diff --git a/tests/st/calicoctl/__init__.py b/tests/st/calicoctl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/calicoctl/test_autodetection.py b/tests/st/calicoctl/test_autodetection.py new file mode 100644 index 000000000..8f50b7ce9 --- /dev/null +++ b/tests/st/calicoctl/test_autodetection.py @@ -0,0 +1,103 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nose.plugins.attrib import attr + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost +from tests.st.utils.utils import ETCD_CA, ETCD_CERT, \ + ETCD_KEY, ETCD_HOSTNAME_SSL, ETCD_SCHEME, get_ip +from tests.st.utils.exceptions import CommandExecError + +if ETCD_SCHEME == "https": + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " \ + "--cluster-store-opt kv.cacertfile=%s " \ + "--cluster-store-opt kv.certfile=%s " \ + "--cluster-store-opt kv.keyfile=%s " % \ + (ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, + ETCD_KEY) +else: + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ + get_ip() + +class TestAutodetection(TestBase): + + @attr('slow') + def test_autodetection(self): + """ + Test using different IP autodetection methods. + + We run a multi-host test for this to test explicit selection of + "first-found" and also "interface" and "can-reach" detection methods. + """ + with DockerHost('host1', + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + start_calico=False) as host1, \ + DockerHost('host2', + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + start_calico=False) as host2, \ + DockerHost('host3', + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + start_calico=False) as host3: + + # Start the node on host1 using first-found auto-detection + # method. + host1.start_calico_node( + "--ip=autodetect --ip-autodetection-method=first-found") + + # Attempt to start the node on host2 using can-reach auto-detection + # method using a bogus DNS name. This should fail. + try: + host2.start_calico_node( + "--ip=autodetect --ip-autodetection-method=can-reach=XXX.YYY.ZZZ.XXX") + except CommandExecError: + pass + else: + raise AssertionError("Command expected to fail but did not") + + # Start the node on host2 using can-reach auto-detection method + # using the IP address of host1. This should succeed. + host2.start_calico_node( + "--ip=autodetect --ip-autodetection-method=can-reach=" + host1.ip) + + # Attempt to start the node on host3 using interface auto-detection + # method using a bogus interface name. This should fail. + try: + host3.start_calico_node( + "--ip=autodetect --ip-autodetection-method=interface=BogusInterface") + except CommandExecError: + pass + else: + raise AssertionError("Command expected to fail but did not") + + # Start the node on host2 using can-reach auto-detection method + # using the IP address of host1. This should succeed. + host3.start_calico_node( + "--ip=autodetect --ip-autodetection-method=interface=eth0") + + # Create a network and a workload on each host. + network1 = host1.create_network("subnet1") + workload_host1 = host1.create_workload("workload1", network=network1) + workload_host2 = host2.create_workload("workload2", network=network1) + workload_host3 = host3.create_workload("workload3", network=network1) + + # Allow network to converge + self.assert_true(workload_host1.check_can_ping(workload_host3.ip, retries=10)) + + # Check connectivity in both directions + self.assert_ip_connectivity(workload_list=[workload_host1, + workload_host2, + workload_host3], + ip_pass_list=[workload_host1.ip, + workload_host2.ip, + workload_host3.ip]) diff --git a/tests/st/calicoctl/test_default_pools.py b/tests/st/calicoctl/test_default_pools.py new file mode 100644 index 000000000..3ec7e0f1c --- /dev/null +++ b/tests/st/calicoctl/test_default_pools.py @@ -0,0 +1,161 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import logging + +import netaddr +import yaml +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS, NODE_CONTAINER_NAME +from tests.st.utils.exceptions import CommandExecError +from tests.st.utils.utils import get_ip, wipe_etcd, retry_until_success + +_log = logging.getLogger(__name__) +_log.setLevel(logging.DEBUG) + + +class TestDefaultPools(TestBase): + @classmethod + def setUpClass(cls): + # First, create a (fake) host to run things in + cls.host = DockerHost("host", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + start_calico=False, + dind=False) + + def setUp(self): + try: + self.host.execute("docker rm -f calico-node") + except CommandExecError: + # Presumably calico-node wasn't running + pass + wipe_etcd(get_ip()) + + @classmethod + def tearDownClass(cls): + cls.host.cleanup() + + @parameterized.expand([ + (False, "CALICO_IPV4POOL_CIDR", "10.0.0.0/27", 0, None, "Too small"), + (False, "CALICO_IPV4POOL_CIDR", "10.0.0.0/32", 0, None, "Too small, but legal CIDR"), + (False, "CALICO_IPV4POOL_CIDR", "10.0.0.0/33", 0, None, "Impossible CIDR"), + (False, "CALICO_IPV4POOL_CIDR", "256.0.0.0/24", 0, None, "Invalid IP"), + (True, "CALICO_IPV4POOL_CIDR", "10.0.0.0/24", 2, None, "Typical non-default pool"), + (True, "CALICO_IPV4POOL_CIDR", "10.0.0.0/26", 2, None, "Smallest legal pool"), + (True, "CALICO_IPV6POOL_CIDR", "fd00::/122", 2, None, "Smallest legal pool"), + (False, "CALICO_IPV6POOL_CIDR", "fd00::/123", 0, None, "Too small"), + (False, "CALICO_IPV6POOL_CIDR", "fd00::/128", 0, None, "Too small, but legal CIDR"), + (False, "CALICO_IPV6POOL_CIDR", "fd00::/129", 0, None, "Impossible CIDR"), + (True, "CALICO_IPV4POOL_CIDR", "10.0.0.0/24", 2, "cross-subnet", "Typ. non-def pool, IPIP"), + (True, "CALICO_IPV4POOL_CIDR", "10.0.0.0/24", 2, "always", "Typ. non-default pool, IPIP"), + (True, "CALICO_IPV4POOL_CIDR", "10.0.0.0/24", 2, "off", "Typical pool, explicitly no IPIP"), + (True, "CALICO_IPV6POOL_CIDR", "fd00::/122", 2, "always", "IPv6 - IPIP not permitted"), + (True, "CALICO_IPV6POOL_CIDR", "fd00::/122", 2, "cross-subnet", "IPv6 - IPIP not allowed"), + (True, "CALICO_IPV6POOL_CIDR", "fd00::/122", 2, "off", "IPv6, IPIP explicitly off"), + (False, "CALICO_IPV6POOL_CIDR", "fd00::/122", 0, "junk", "Invalid IPIP value"), + (False, "CALICO_IPV4POOL_CIDR", "10.0.0.0/24", 0, "reboot", "Invalid IPIP value"), + (False, "CALICO_IPV4POOL_CIDR", "0.0.0.0/0", 0, None, "Invalid, link local address"), + (False, "CALICO_IPV6POOL_CIDR", "::/0", 0, None, "Invalid, link local address"), + (True, "CALICO_IPV6POOL_CIDR", "fd80::0:0/120", 2, None, "Valid, but non-canonical form"), + (False, "CALICO_IPV6POOL_CIDR", "1.2.3.4/24", 0, None, "Wrong type"), + (False, "CALICO_IPV4POOL_CIDR", "fd00::/24", 0, None, "Wrong type"), + (True, "CALICO_IPV6POOL_CIDR", "::0:a:b:c:d:e:0/120", 2, None, "Valid, non-canonical form"), + (False, "CALICO_IPV4POOL_CIDR", "1.2/16", 0, None, "Valid, unusual form"), + ]) + def test_default_pools(self, success_expected, param, value, exp_num_pools, ipip, description): + """ + Test that the various options for default pools work correctly + """ + _log.debug("Test description: %s", description) + # Get command line for starting docker + output = self.host.calicoctl("node run --dryrun --node-image=%s" % NODE_CONTAINER_NAME) + base_command = output.split('\n')[-4].rstrip() + + # Modify command line to add the options we want to test + env_inserts = "-e %s=%s " % (param, value) + if ipip is not None: + env_inserts += "-e CALICO_IPV4POOL_IPIP=%s " % ipip + prefix, _, suffix = base_command.partition("-e") + command = prefix + env_inserts + "-e" + suffix + + # Start calico-docker + self.host.execute(command) + + if not success_expected: + # check for "Calico node failed to start" + self.wait_for_node_log("Calico node failed to start") + return + + # Check we started OK + self.wait_for_node_log("Calico node started successfully") + # check the expected pool is present + pools_output = self.host.calicoctl("get ippool -o yaml") + pools_dict = yaml.safe_load(pools_output) + cidrs = [pool['metadata']['cidr'] for pool in pools_dict] + # Convert to canonical form + value = str(netaddr.IPNetwork(value)) + assert value in cidrs, "Didn't find %s in %s" % (value, cidrs) + + # Dump pools and attempt to load them with calicoctl (to confirm consistency) + self.host.calicoctl("get ippool -o yaml > testfile.yaml") + self.host.calicoctl("apply -f testfile.yaml") + + assert len(pools_dict) == exp_num_pools, \ + "Expected %s pools, found %s. %s" % (exp_num_pools, len(pools_dict), pools_dict) + + # Grab the pool of interest + pool = pools_dict[cidrs.index(value)] + other_pool = None + # And grab the other pool if any + if len(pools_dict) > 1: + pools_dict.remove(pool) + other_pool = pools_dict[0] + # Check IPIP setting if we're doing IPv4 + if ipip in ["cross-subnet", "always"] and param == "CALICO_IPV4POOL_CIDR": + assert pool['spec']['ipip']['enabled'] is True, \ + "Didn't find ipip enabled in pool %s" % pool + assert pool['spec']['ipip']['mode'] == ipip, \ + "Didn't find ipip mode in pool %s" % pool + if ipip in [None, "off"] or param == "CALICO_IPV6POOL_CIDR": + assert 'ipip' not in pool['spec'] + if ipip in ["cross-subnet", "always"] and param == "CALICO_IPV6POOL_CIDR": + assert other_pool['spec']['ipip']['enabled'] is True, \ + "Didn't find ipip enabled in pool %s" % pool + assert other_pool['spec']['ipip']['mode'] == ipip, \ + "Didn't find ipip mode in pool %s" % pool + + # Check NAT setting + assert pool['spec']['nat-outgoing'] is True, "Didn't find nat enabled in pool %s" % pool + + def test_no_default_pools(self): + """ + Test that NO_DEFAULT_POOLS works correctly + """ + # Start calico-docker + self.host.start_calico_node(options="--no-default-ippools") + self.wait_for_node_log("Calico node started successfully") + # check the expected pool is present + pools_output = self.host.calicoctl("get ippool -o yaml") + pools_dict = yaml.safe_load(pools_output) + assert pools_dict == [], "Pools not empty: %s" % pools_dict + + def assert_calico_node_log_contains(self, expected_string): + assert expected_string in self.host.execute("docker logs calico-node"), \ + "Didn't find %s in start log" % expected_string + + def wait_for_node_log(self, expected_log): + check = functools.partial(self.assert_calico_node_log_contains, expected_log) + retry_until_success(check, 5, ex_class=AssertionError) diff --git a/tests/st/calicoctl/test_node_checksystem.py b/tests/st/calicoctl/test_node_checksystem.py new file mode 100644 index 000000000..0a7ce1f75 --- /dev/null +++ b/tests/st/calicoctl/test_node_checksystem.py @@ -0,0 +1,31 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost + +""" +Test calicoctl checksystem + +It's worth doing a simple return code check. Anything more is going to be +difficult given the environmental requirements. +""" + + +class TestNodeCheckSystem(TestBase): + def test_node_checksystem(self): + """ + Test that the checksystem command can be executed. + """ + with DockerHost('host', dind=False, start_calico=False) as host: + host.calicoctl("node checksystem") diff --git a/tests/st/calicoctl/test_node_diags.py b/tests/st/calicoctl/test_node_diags.py new file mode 100644 index 000000000..21c4cacec --- /dev/null +++ b/tests/st/calicoctl/test_node_diags.py @@ -0,0 +1,38 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost + +""" +Test calicoctl diags. + +It's worth testing that the command can be executed. It's debatable whether +it's worth testing the upload. + +We're not trying to assert on the contents of the diags package. + +TODO We could check that the file is actually written (and doesn't just appear +in the output) and is a decent size. +TODO We could check collecting diags when calico-node is actually running. +""" + + +class TestNodeDiags(TestBase): + def test_node_diags(self): + """ + Test that the diags command successfully creates a tar.gz file. + """ + with DockerHost('host', dind=False, start_calico=False) as host: + results = host.calicoctl("node diags") + self.assertIn(".tar.gz", results) diff --git a/tests/st/calicoctl/test_node_run.py b/tests/st/calicoctl/test_node_run.py new file mode 100644 index 000000000..8660fd5f9 --- /dev/null +++ b/tests/st/calicoctl/test_node_run.py @@ -0,0 +1,31 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost + +""" +Test calicoctl node run + +""" + +class TestNodeRun(TestBase): + def test_node_run_dryrun(self): + """ + Test that dryrun does not output ETCD_AUTHORITY or ETCD_SCHEME. + """ + with DockerHost('host', dind=False, start_calico=False) as host: + output = host.calicoctl("node run --dryrun") + assert "ETCD_AUTHORITY" not in output + assert "ETCD_SCHEME" not in output + assert "ETCD_ENDPOINTS" in output diff --git a/tests/st/calicoctl/test_node_status.py b/tests/st/calicoctl/test_node_status.py new file mode 100644 index 000000000..48dd76b45 --- /dev/null +++ b/tests/st/calicoctl/test_node_status.py @@ -0,0 +1,49 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost +from tests.st.utils.exceptions import CommandExecError +from tests.st.utils.utils import retry_until_success + +""" +Test calicoctl status + +Most of the status output is checked by the BGP tests, so this module just +contains a simple return code check. +""" + +class TestNodeStatus(TestBase): + def test_node_status(self): + """ + Test that the status command can be executed. + """ + with DockerHost('host', dind=False, start_calico=True) as host: + def node_status(): + host.calicoctl("node status") + retry_until_success(node_status, retries=10, ex_class=Exception) + + def test_node_status_fails(self): + """ + Test that the status command fails when calico node is not running + """ + with DockerHost('host', dind=False, start_calico=False) as host: + try: + host.calicoctl("node status") + except CommandExecError as e: + self.assertEquals(e.returncode, 1) + self.assertEquals(e.output, + "Calico process is not running.\n") + else: + raise AssertionError("'calicoctl status' did not exit with " + "code 1 when node was not running") diff --git a/tests/st/ipam/__init__.py b/tests/st/ipam/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/ipam/test_ipam.py b/tests/st/ipam/test_ipam.py new file mode 100644 index 000000000..36e3b833c --- /dev/null +++ b/tests/st/ipam/test_ipam.py @@ -0,0 +1,225 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import random + +import netaddr +import yaml +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS + +POST_DOCKER_COMMANDS = ["docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar", + "docker load -i /code/workload.tar"] + +logging.basicConfig(level=logging.DEBUG, format="%(message)s") +logger = logging.getLogger(__name__) + + +class MultiHostIpam(TestBase): + @classmethod + def setUpClass(cls): + super(TestBase, cls).setUpClass() + cls.hosts = [] + cls.hosts.append(DockerHost("host1", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False)) + cls.hosts.append(DockerHost("host2", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False)) + cls.hosts[0].start_calico_node() + cls.hosts[1].start_calico_node() + cls.network = cls.hosts[0].create_network("testnet1", ipam_driver="calico-ipam") + + @classmethod + def tearDownClass(cls): + # Tidy up + cls.network.delete() + for host in cls.hosts: + host.cleanup() + del host + + def setUp(self): + # Save off original pool if any, then wipe pools so we have a known ground state + response = self.hosts[0].calicoctl("get IPpool -o yaml") + self.orig_pools = yaml.safe_load(response) + if len(self.orig_pools) > 0: + self.hosts[0].writefile("orig_pools.yaml", response) + self.hosts[0].calicoctl("delete -f orig_pools.yaml") + + def tearDown(self): + # Replace original pool, if any + if len(self.orig_pools) > 0: + self.hosts[0].calicoctl("apply -f orig_pools.yaml") + # Remove all workloads + for host in self.hosts: + host.remove_workloads() + + def test_pools_add(self): + """ + (Add a pool), create containers, check IPs assigned from pool. + Then Delete that pool. + Add a new pool, create containers, check IPs assigned from NEW pool + """ + old_pool_workloads = [] + ipv4_subnet = netaddr.IPNetwork("192.168.0.0/24") + new_pool = {'apiVersion': 'v1', + 'kind': 'ipPool', + 'metadata': {'cidr': str(ipv4_subnet.ipv4())}, + } + self.hosts[0].writefile("newpool.yaml", yaml.dump(new_pool)) + self.hosts[0].calicoctl("create -f newpool.yaml") + + for host in self.hosts: + workload = host.create_workload("wlda-%s" % host.name, + image="workload", + network=self.network) + assert netaddr.IPAddress(workload.ip) in ipv4_subnet + old_pool_workloads.append((workload, host)) + + blackhole_cidr = netaddr.IPNetwork( + self.hosts[0].execute("ip r | grep blackhole").split()[1]) + assert blackhole_cidr in ipv4_subnet + # Check there's only one /32 present and that its within the pool + output = self.hosts[0].execute("ip r | grep cali").split('\n') + assert len(output) == 1, "Output should only be 1 line. Got: %s" % output + wl_ip = netaddr.IPNetwork(output[0].split()[0]) + assert wl_ip in ipv4_subnet + + self.hosts[0].calicoctl("delete -f newpool.yaml") + + ipv4_subnet = netaddr.IPNetwork("10.0.1.0/24") + new_pool = {'apiVersion': 'v1', + 'kind': 'ipPool', + 'metadata': {'cidr': str(ipv4_subnet.ipv4())}, + } + self.hosts[0].writefile("pools.yaml", yaml.dump(new_pool)) + self.hosts[0].calicoctl("create -f pools.yaml") + + self.hosts[0].remove_workloads() + + for host in self.hosts: + workload = host.create_workload("wlda2-%s" % host.name, + image="workload", + network=self.network) + assert netaddr.IPAddress(workload.ip) in ipv4_subnet, \ + "Workload IP in wrong pool. IP: %s, Pool: %s" % (workload.ip, ipv4_subnet.ipv4()) + + blackhole_cidr = netaddr.IPNetwork( + self.hosts[0].execute("ip r | grep blackhole").split()[1]) + assert blackhole_cidr in ipv4_subnet + # Check there's only one /32 present and that its within the pool + output = self.hosts[0].execute("ip r | grep cali").split('\n') + assert len(output) == 1, "Output should only be 1 line. Got: %s" % output + wl_ip = netaddr.IPNetwork(output[0].split()[0]) + assert wl_ip in ipv4_subnet + + def test_ipam_show(self): + """ + Create some workloads, then ask calicoctl to tell you about the IPs in the pool. + Check that the correct IPs are shown as in use. + """ + num_workloads = 10 + workload_ips = [] + + ipv4_subnet = netaddr.IPNetwork("192.168.45.0/25") + new_pool = {'apiVersion': 'v1', + 'kind': 'ipPool', + 'metadata': {'cidr': str(ipv4_subnet.ipv4())}, + } + self.hosts[0].writefile("newpool.yaml", yaml.dump(new_pool)) + self.hosts[0].calicoctl("create -f newpool.yaml") + + for i in range(num_workloads): + host = random.choice(self.hosts) + workload = host.create_workload("wlds-%s" % i, + image="workload", + network=self.network) + workload_ips.append(workload.ip) + + print workload_ips + + for ip in ipv4_subnet: + response = self.hosts[0].calicoctl("ipam show --ip=%s" % ip) + if "No attributes defined for" in response: + # This means the IP is assigned + assert str(ip) in workload_ips, "ipam show says IP %s " \ + "is assigned when it is not" % ip + if "not currently assigned in block" in response: + # This means the IP is not assigned + assert str(ip) not in workload_ips, \ + "ipam show says IP %s is not assigned when it is!" % ip + + @parameterized.expand([ + (False,), + (True,), + ]) + def test_pool_wrap(self, make_static_workload): + """ + Repeatedly create and delete workloads until the system re-assigns an IP. + """ + + ipv4_subnet = netaddr.IPNetwork("192.168.46.0/25") + new_pool = {'apiVersion': 'v1', + 'kind': 'ipPool', + 'metadata': {'cidr': str(ipv4_subnet.ipv4())}, + } + self.hosts[0].writefile("newpool.yaml", yaml.dump(new_pool)) + self.hosts[0].calicoctl("create -f newpool.yaml") + + host = self.hosts[0] + i = 0 + if make_static_workload: + static_workload = host.create_workload("static", + image="workload", + network=self.network) + i += 1 + + new_workload = host.create_workload("wldw-%s" % i, + image="workload", + network=self.network) + assert netaddr.IPAddress(new_workload.ip) in ipv4_subnet + original_ip = new_workload.ip + while True: + self.delete_workload(host, new_workload) + i += 1 + new_workload = host.create_workload("wldw-%s" % i, + image="workload", + network=self.network) + assert netaddr.IPAddress(new_workload.ip) in ipv4_subnet + if make_static_workload: + assert new_workload.ip != static_workload.ip, "IPAM assigned an IP which is " \ + "still in use!" + + if new_workload.ip == original_ip: + # We assign pools to hosts in /26's - so 64 addresses. + poolsize = 64 + # But if we're using one for a static workload, there will be one less + if make_static_workload: + poolsize -= 1 + assert i >= poolsize, "Original IP was re-assigned before entire host pool " \ + "was cycled through. Hit after %s times" % i + break + if i > (len(ipv4_subnet) * 2): + assert False, "Cycled twice through pool - original IP still not assigned." + + @staticmethod + def delete_workload(host, workload): + host.calicoctl("ipam release --ip=%s" % workload.ip) + host.execute("docker rm -f %s" % workload.name) + host.workloads.remove(workload) diff --git a/tests/st/libnetwork/test_labeling.py b/tests/st/libnetwork/test_labeling.py new file mode 100644 index 000000000..3cc3c9a7e --- /dev/null +++ b/tests/st/libnetwork/test_labeling.py @@ -0,0 +1,177 @@ +# Copyright (c) 2017 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import yaml +from nose_parameterized import parameterized +from unittest import skip + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost +from tests.st.utils.utils import ETCD_CA, ETCD_CERT, \ + ETCD_KEY, ETCD_HOSTNAME_SSL, ETCD_SCHEME, get_ip, \ + retry_until_success, wipe_etcd + +POST_DOCKER_COMMANDS = [ + "docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar", + "docker load -i /code/workload.tar", +] + +if ETCD_SCHEME == "https": + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " \ + "--cluster-store-opt kv.cacertfile=%s " \ + "--cluster-store-opt kv.certfile=%s " \ + "--cluster-store-opt kv.keyfile=%s " % \ + (ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, + ETCD_KEY) +else: + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ + get_ip() + + +class TestLibnetworkLabeling(TestBase): + """ + Tests that labeling is correctly implemented in libnetwork. Setup + multiple networks and then run containers with labels and see that + policy will allow and block traffic. + """ + hosts = None + host = None + + @classmethod + def setUpClass(cls): + wipe_etcd(get_ip()) + + # Rough idea for setup + # + # Network1 Network2 + # + # container1 container2 + # foo = bar baz = bop + # + # container3 container4 + # foo = bing foo = bar + + cls.hosts = [] + cls.host1 = DockerHost( + "host1", + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) + cls.host1_hostname = cls.host1.execute("hostname") + cls.hosts.append(cls.host1) + cls.host2 = DockerHost( + "host2", + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) + cls.host2_hostname = cls.host1.execute("hostname") + cls.hosts.append(cls.host2) + + for host in cls.hosts: + host.start_calico_node(options='--use-docker-networking-container-labels') + + cls.network1 = cls.host1.create_network("network1") + cls.network2 = cls.host1.create_network("network2") + + cls.workload1_nw1_foo_bar = cls.host1.create_workload( + "workload1", network=cls.network1, + labels=["org.projectcalico.label.foo=bar"]) + cls.workload2_nw2_baz_bop = cls.host1.create_workload( + "workload2", network=cls.network2, + labels=["org.projectcalico.label.baz=bop"]) + cls.workload3_nw1_foo_bing = cls.host2.create_workload( + "workload3", network=cls.network1, + labels=["org.projectcalico.label.foo=bing"]) + cls.workload4_nw2_foo_bar = cls.host2.create_workload( + "workload4", network=cls.network2, + labels=["org.projectcalico.label.foo=bar"]) + + def setUp(self): + # Override the per-test setUp to avoid wiping etcd; instead only + # clean up the data we added. + self.host1.delete_all_resource("policy") + + def tearDown(self): + self.host1.delete_all_resource("policy") + super(TestLibnetworkLabeling, self).tearDown() + + @classmethod + def tearDownClass(cls): + # Tidy up + for host in cls.hosts: + host.remove_workloads() + for host in cls.hosts: + host.cleanup() + del host + + def test_policy_only_selectors_allow_traffic(self): + self.host1.add_resource([ + { + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'allowFooBarToBazBop'}, + 'spec': { + 'ingress': [ + { + 'source': {'selector': 'foo == "bar"'}, + 'action': 'allow', + }, + ], + 'egress': [{'action': 'deny'}], + 'selector': 'baz == "bop"' + } + }, { + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'allowFooBarEgress'}, + 'spec': { + 'selector': 'foo == "bar"', + 'egress': [{'action': 'allow'}] + } + } + ]) + + retry_until_success(lambda: self.assert_ip_connectivity( + workload_list=[self.workload1_nw1_foo_bar, + self.workload4_nw2_foo_bar], + ip_pass_list=[self.workload2_nw2_baz_bop.ip], + ip_fail_list=[self.workload3_nw1_foo_bing.ip]), 3) + + def test_no_policy_allows_no_traffic(self): + retry_until_success(lambda: self.assert_ip_connectivity( + workload_list=[self.workload1_nw1_foo_bar, + self.workload2_nw2_baz_bop, + self.workload3_nw1_foo_bing], + ip_pass_list=[], + ip_fail_list=[self.workload4_nw2_foo_bar.ip]), 2) + retry_until_success(lambda: self.assert_ip_connectivity( + workload_list=[self.workload2_nw2_baz_bop, + self.workload3_nw1_foo_bing, + self.workload4_nw2_foo_bar], + ip_pass_list=[], + ip_fail_list=[self.workload1_nw1_foo_bar.ip]), 2) + retry_until_success(lambda: self.assert_ip_connectivity( + workload_list=[self.workload1_nw1_foo_bar, + self.workload3_nw1_foo_bing, + self.workload4_nw2_foo_bar], + ip_pass_list=[], + ip_fail_list=[self.workload2_nw2_baz_bop.ip]), 2) + retry_until_success(lambda: self.assert_ip_connectivity( + workload_list=[self.workload1_nw1_foo_bar, + self.workload2_nw2_baz_bop, + self.workload4_nw2_foo_bar], + ip_pass_list=[], + ip_fail_list=[self.workload3_nw1_foo_bing.ip]), 2) diff --git a/tests/st/policy/__init__.py b/tests/st/policy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/policy/test_felix_gateway.py b/tests/st/policy/test_felix_gateway.py new file mode 100644 index 000000000..0c2136b18 --- /dev/null +++ b/tests/st/policy/test_felix_gateway.py @@ -0,0 +1,493 @@ +# Copyright (c) 2017 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import subprocess +import yaml +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.utils import get_ip, log_and_run, retry_until_success, \ + ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL, ETCD_SCHEME + +_log = logging.getLogger(__name__) +_log.setLevel(logging.DEBUG) + +POST_DOCKER_COMMANDS = [ + "docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar", + "docker load -i /code/workload.tar", +] + +class TestFelixOnGateway(TestBase): + """ + Tests that policy is correctly implemented when using Calico + on a gateway or router. In that scenario, Calico should + police forwarded (possibly NATted) traffic using the host endpoint + policy. + """ + hosts = None + gateway = None + host = None + + @classmethod + def setUpClass(cls): + # Wipe etcd once before any test in this class runs. + wipe_etcd() + + # We set up an additional docker network to act as the external + # network. The Gateway container is connected to both networks. + # and we configure it as a NAT gateway. + # + # "cali-st-ext" host + # container + # | + # "cali-st-ext" docker + # bridge + # | + # Gateway Host + # container container + # \ / + # default docker + # bridge + + # First, create the hosts and the gateway. + cls.hosts = [] + cls.gateway = DockerHost("cali-st-gw", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) + cls.gateway_hostname = cls.gateway.execute("hostname") + cls.host = DockerHost("cali-st-host", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) + cls.host_hostname = cls.host.execute("hostname") + cls.hosts.append(cls.gateway) + cls.hosts.append(cls.host) + + # Delete the nginx container if it still exists. We need to do this + # before we try to remove the network. + log_and_run("docker rm -f cali-st-ext-nginx || true") + + # Create the external network. + log_and_run("docker network rm cali-st-ext || true") + # Use 172.19.0.0 to avoid clash with normal docker subnet and + # docker-in-docker subnet + log_and_run("docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext") + + # And an nginx server on the external network only. + log_and_run("docker run" + " --network=cali-st-ext" + " -d" + " --name=cali-st-ext-nginx" + " nginx") + + for host in cls.hosts: + host.start_calico_node() + + # Get the internal IP of the gateway. We do this before we add the second + # network since it means we don't have to figure out which IP is which. + int_ip = str(cls.gateway.ip) + cls.gateway_int_ip = int_ip + _log.info("Gateway internal IP: %s", cls.gateway_int_ip) + + # Add the gateway to the external network. + log_and_run("docker network connect cali-st-ext cali-st-gw") + cls.gateway.execute("ip addr") + + # Get the IP of the external server. + ext_ip = cls.get_container_ip("cali-st-ext-nginx") + cls.ext_server_ip = ext_ip + _log.info("External workload IP: %s", cls.ext_server_ip) + + # Configure the internal host to use the gateway for the external IP. + cls.host.execute("ip route add %s via %s" % + (cls.ext_server_ip, cls.gateway_int_ip)) + + # Configure the gateway to forward and NAT. + cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1") + cls.gateway.execute("iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" % + cls.ext_server_ip) + + def setUp(self): + # Override the per-test setUp to avoid wiping etcd; instead only clean up the data we + # added. + self.remove_pol_and_endpoints() + + def tearDown(self): + self.remove_pol_and_endpoints() + super(TestFelixOnGateway, self).tearDown() + + @classmethod + def tearDownClass(cls): + # Tidy up + for host in cls.hosts: + host.remove_workloads() + for host in cls.hosts: + host.cleanup() + del host + + log_and_run("docker rm -f cali-st-ext-nginx || true") + + def test_ingress_policy_can_block_through_traffic(self): + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-int'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'protocol': 'tcp', + 'destination': {'ports': [80]}, + 'action': 'deny' + }, + ], + 'egress': [ + {'action': 'deny'}, + ], + 'selector': 'role == "gateway-int"' + } + }) + self.add_gateway_internal_iface() + retry_until_success(self.assert_host_can_not_curl_ext, 3) + + def test_ingress_policy_can_allow_through_traffic(self): + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-int'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'protocol': 'tcp', + 'destination': {'ports': [80]}, + 'action': 'allow' + }, + ], + 'egress': [ + {'action': 'deny'}, + ], + 'selector': 'role == "gateway-int"' + } + }) + self.add_gateway_internal_iface() + retry_until_success(self.assert_host_can_curl_ext, 3) + + def test_egress_policy_can_block_through_traffic(self): + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-ext'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'action': 'deny', + }, + ], + 'egress': [ + { + 'protocol': 'tcp', + 'destination': {'ports': [80]}, + 'action': 'deny' + }, + ], + 'selector': 'role == "gateway-ext"' + } + }) + self.add_gateway_external_iface() + retry_until_success(self.assert_host_can_not_curl_ext, 3) + + def test_egress_policy_can_allow_through_traffic(self): + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-ext'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'action': 'deny', + }, + ], + 'egress': [ + { + 'protocol': 'tcp', + 'destination': {'ports': [80]}, + 'action': 'allow' + }, + ], + 'selector': 'role == "gateway-ext"' + } + }) + self.add_gateway_external_iface() + retry_until_success(self.assert_host_can_curl_ext, 3) + + def test_ingress_and_egress_policy_can_allow_through_traffic(self): + self.add_gateway_external_iface() + self.add_gateway_internal_iface() + self.add_host_iface() + + # Adding the host endpoints should break connectivity until we add policy back in. + retry_until_success(self.assert_host_can_not_curl_ext, 3) + + # Add in the policy... + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'host-out'}, + 'spec': { + 'order': 10, + 'selector': 'role == "host"', + 'egress': [{'action': 'allow'}], + 'ingress': [{'action': 'allow'}], + } + }) + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-int'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'protocol': 'tcp', + 'destination': { + 'ports': [80], + 'net': self.ext_server_ip + "/32", + }, + 'source': { + 'selector': 'role == "host"', + }, + 'action': 'allow' + }, + ], + 'egress': [], + 'selector': 'role == "gateway-int"' + } + }) + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-ext'}, + 'spec': { + 'order': 10, + 'ingress': [], + 'egress': [ + { + 'protocol': 'tcp', + 'destination': { + 'ports': [80], + 'net': self.ext_server_ip + "/32", + }, + 'source': { + 'selector': 'role == "host"', + }, + 'action': 'allow' + }, + ], + 'selector': 'role == "gateway-ext"' + } + }) + + retry_until_success(self.assert_host_can_curl_ext, 3) + + @parameterized.expand([ + ('allow', 'deny'), + ('deny', 'allow') + ]) + def test_conflicting_ingress_and_egress_policy(self, in_action, out_action): + # If there is policy on the ingress and egress interface then both should + # get applied and 'deny' should win. + self.add_host_iface() + self.add_gateway_external_iface() + self.add_gateway_internal_iface() + + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-int'}, + 'spec': { + 'order': 10, + 'ingress': [ + { + 'action': in_action + }, + ], + 'egress': [], + 'selector': 'role == "gateway-int"' + } + }) + self.add_policy({ + 'apiVersion': 'v1', + 'kind': 'policy', + 'metadata': {'name': 'port80-ext'}, + 'spec': { + 'order': 10, + 'ingress': [], + 'egress': [ + { + 'action': out_action + }, + ], + 'selector': 'role == "gateway-ext"' + } + }) + retry_until_success(self.assert_host_can_not_curl_ext, 3) + + def add_policy(self, policy_data): + self._apply_resources(policy_data, self.gateway) + + def add_gateway_internal_iface(self): + host_endpoint_data = { + 'apiVersion': 'v1', + 'kind': 'hostEndpoint', + 'metadata': { + 'name': 'gw-int', + 'node': self.gateway_hostname, + 'labels': {'role': 'gateway-int'} + }, + 'spec': { + 'interfaceName': 'eth0' + } + } + self._apply_resources(host_endpoint_data, self.gateway) + + def add_gateway_external_iface(self): + host_endpoint_data = { + 'apiVersion': 'v1', + 'kind': 'hostEndpoint', + 'metadata': { + 'name': 'gw-ext', + 'node': self.gateway_hostname, + 'labels': {'role': 'gateway-ext'} + }, + 'spec': { + 'interfaceName': 'eth1' + } + } + self._apply_resources(host_endpoint_data, self.gateway) + + def add_host_iface(self): + host_endpoint_data = { + 'apiVersion': 'v1', + 'kind': 'hostEndpoint', + 'metadata': { + 'name': 'host-int', + 'node': self.host_hostname, + 'labels': {'role': 'host'} + }, + 'spec': { + 'interfaceName': 'eth0', + 'expectedIPs': [str(self.host.ip)], + } + } + self._apply_resources(host_endpoint_data, self.gateway) + + def assert_host_can_curl_ext(self): + try: + self.host.execute("curl --fail -m 1 -o /tmp/nginx-index.html %s" % self.ext_server_ip) + except subprocess.CalledProcessError: + _log.exception("Internal host failed to curl external server IP: %s", + self.ext_server_ip) + self.fail("Internal host failed to curl external server IP: %s" % self.ext_server_ip) + + def assert_host_can_not_curl_ext(self): + try: + self.host.execute("curl --fail -m 1 -o /tmp/nginx-index.html %s" % self.ext_server_ip) + except subprocess.CalledProcessError: + return + else: + self.fail("Internal host can curl external server IP: %s" % self.ext_server_ip) + + def remove_pol_and_endpoints(self): + self.delete_all("pol") + self.delete_all("hostEndpoint") + # Wait for felix to remove the policy and allow traffic through the gateway. + retry_until_success(self.assert_host_can_curl_ext) + + def delete_all(self, resource): + # Grab all objects of a resource type + objects = yaml.load(self.hosts[0].calicoctl("get %s -o yaml" % resource)) + # and delete them (if there are any) + if len(objects) > 0: + self._delete_data(objects, self.hosts[0]) + + def _delete_data(self, data, host): + _log.debug("Deleting data with calicoctl: %s", data) + self._exec_calicoctl("delete", data, host) + + @classmethod + def _apply_resources(cls, resources, host): + cls._exec_calicoctl("apply", resources, host) + + @staticmethod + def _exec_calicoctl(action, data, host): + # use calicoctl with data + host.writefile("new_data", + yaml.dump(data, default_flow_style=False)) + host.calicoctl("%s -f new_data" % action) + + @classmethod + def get_container_ip(cls, container_name): + ip = log_and_run( + "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s" % + container_name) + return ip.strip() + + +def wipe_etcd(): + _log.debug("Wiping etcd") + # Delete /calico if it exists. This ensures each test has an empty data + # store at start of day. + curl_etcd(get_ip(), "calico", options=["-XDELETE"]) + + # Disable Usage Reporting to usage.projectcalico.org + # We want to avoid polluting analytics data with unit test noise + curl_etcd(get_ip(), + "calico/v1/config/UsageReportingEnabled", + options=["-XPUT -d value=False"]) + curl_etcd(get_ip(), + "calico/v1/config/LogSeverityScreen", + options=["-XPUT -d value=debug"]) + + +def curl_etcd(ip, path, options=None, recursive=True): + """ + Perform a curl to etcd, returning JSON decoded response. + :param ip: IP address of etcd server + :param path: The key path to query + :param options: Additional options to include in the curl + :param recursive: Whether we want recursive query or not + :return: The JSON decoded response. + """ + if options is None: + options = [] + if ETCD_SCHEME == "https": + # Etcd is running with SSL/TLS, require key/certificates + command = "curl --cacert %s --cert %s --key %s " \ + "-sL https://%s:2379/v2/keys/%s?recursive=%s %s" % \ + (ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL, path, + str(recursive).lower(), " ".join(options)) + else: + command = "curl -sL http://%s:2379/v2/keys/%s?recursive=%s %s" % \ + (ip, path, str(recursive).lower(), " ".join(options)) + _log.debug("Running: %s", command) + rc = subprocess.check_output(command, shell=True) + return json.loads(rc.strip()) diff --git a/tests/st/policy/test_profile.py b/tests/st/policy/test_profile.py new file mode 100644 index 000000000..46b2fff9d --- /dev/null +++ b/tests/st/policy/test_profile.py @@ -0,0 +1,327 @@ +# Copyright 2015 Tigera, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import netaddr +import yaml +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost, CLUSTER_STORE_DOCKER_OPTIONS +from tests.st.utils.exceptions import CommandExecError +from tests.st.utils.utils import assert_network, assert_profile, \ + assert_number_endpoints, get_profile_name + +POST_DOCKER_COMMANDS = ["docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar", + "docker load -i /code/workload.tar"] + + +class MultiHostMainline(TestBase): + @parameterized.expand([ + #"tags", + "rules.tags", + #"rules.protocol.icmp", + #"rules.ip.addr", + #"rules.ip.net", + #"rules.selector", + #"rules.tcp.port", + #"rules.udp.port", + ]) + def test_multi_host(self, test_type): + """ + Run a mainline multi-host test. + Because multihost tests are slow to setup, this tests most mainline + functionality in a single test. + - Create two hosts + - Create a network using the default IPAM driver, and a workload on + each host assigned to that network. + - Create a network using the Calico IPAM driver, and a workload on + each host assigned to that network. + - Check that hosts on the same network can ping each other. + - Check that hosts on different networks cannot ping each other. + - Modify the profile rules + - Check that connectivity has changed to match the profile we set up + - Re-apply the original profile + - Check that connectivity goes back to what it was originally. + """ + with DockerHost("host1", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) as host1, \ + DockerHost("host2", + additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False) as host2: + (n1_workloads, n2_workloads, networks) = \ + self._setup_workloads(host1, host2) + + # Get the original profiles: + output = host1.calicoctl("get profile -o yaml") + original_profiles = yaml.safe_load(output) + # Make a copy of the profiles to mess about with. + new_profiles = copy.deepcopy(original_profiles) + + if test_type == "tags": + profile0_tag = new_profiles[0]['metadata']['tags'][0] + profile1_tag = new_profiles[1]['metadata']['tags'][0] + # Make a new profiles dict where the two networks have each + # other in their tags list + new_profiles[0]['metadata']['tags'].append(profile1_tag) + new_profiles[1]['metadata']['tags'].append(profile0_tag) + + self._apply_new_profile(new_profiles, host1) + # Check everything can contact everything else now + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.tags": + profile0_tag = new_profiles[0]['metadata']['tags'][0] + profile1_tag = new_profiles[1]['metadata']['tags'][0] + rule0 = {'action': 'allow', + 'source': + {'tag': profile1_tag}} + rule1 = {'action': 'allow', + 'source': + {'tag': profile0_tag}} + new_profiles[0]['spec']['ingress'].append(rule0) + new_profiles[1]['spec']['ingress'].append(rule1) + self._apply_new_profile(new_profiles, host1) + # Check everything can contact everything else now + self.assert_connectivity(retries=3, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.protocol.icmp": + rule = {'action': 'allow', + 'source': + {'protocol': 'icmp'}} + # The copy.deepcopy(rule) is needed to ensure that we don't + # end up with a yaml document with a reference to the same + # rule. While this is probably legal, it isn't main line. + new_profiles[0]['spec']['ingress'].append(rule) + new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule)) + self._apply_new_profile(new_profiles, host1) + # Check everything can contact everything else now + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.ip.addr": + prof_n1, prof_n2 = self._get_profiles(new_profiles) + for workload in n1_workloads: + ip = workload.ip + rule = {'action': 'allow', + 'source': + {'net': '%s/32' % ip}} + prof_n2['spec']['ingress'].append(rule) + for workload in n2_workloads: + ip = workload.ip + rule = {'action': 'allow', + 'source': + {'net': '%s/32' % ip}} + prof_n1['spec']['ingress'].append(rule) + self._apply_new_profile(new_profiles, host1) + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.ip.net": + prof_n1, prof_n2 = self._get_profiles(new_profiles) + n1_ips = [workload.ip for workload in n1_workloads] + n2_ips = [workload.ip for workload in n2_workloads] + n1_subnet = netaddr.spanning_cidr(n1_ips) + n2_subnet = netaddr.spanning_cidr(n2_ips) + rule = {'action': 'allow', + 'source': + {'net': str(n1_subnet)}} + prof_n2['spec']['ingress'].append(rule) + rule = {'action': 'allow', + 'source': + {'net': str(n2_subnet)}} + prof_n1['spec']['ingress'].append(rule) + self._apply_new_profile(new_profiles, host1) + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.selector": + new_profiles[0]['metadata']['labels'] = {'net': 'n1'} + new_profiles[1]['metadata']['labels'] = {'net': 'n2'} + rule = {'action': 'allow', + 'source': + {'selector': 'net=="n2"'}} + new_profiles[0]['spec']['ingress'].append(rule) + rule = {'action': 'allow', + 'source': + {'selector': "net=='n1'"}} + new_profiles[1]['spec']['ingress'].append(rule) + self._apply_new_profile(new_profiles, host1) + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads) + + elif test_type == "rules.tcp.port": + rule = {'action': 'allow', + 'protocol': 'tcp', + 'destination': + {'ports': [80]}} + # The copy.deepcopy(rule) is needed to ensure that we don't + # end up with a yaml document with a reference to the same + # rule. While this is probably legal, it isn't main line. + new_profiles[0]['spec']['ingress'].append(rule) + new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule)) + self._apply_new_profile(new_profiles, host1) + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads, + type_list=['tcp']) + self.assert_connectivity(retries=2, + pass_list=n1_workloads, + fail_list=n2_workloads, + type_list=['icmp', 'udp']) + + elif test_type == "rules.udp.port": + rule = {'action': 'allow', + 'protocol': 'udp', + 'destination': + {'ports': [69]}} + # The copy.deepcopy(rule) is needed to ensure that we don't + # end up with a yaml document with a reference to the same + # rule. While this is probably legal, it isn't main line. + new_profiles[0]['spec']['ingress'].append(rule) + new_profiles[1]['spec']['ingress'].append(copy.deepcopy(rule)) + self._apply_new_profile(new_profiles, host1) + self.assert_connectivity(retries=2, + pass_list=n1_workloads + n2_workloads, + type_list=['udp']) + self.assert_connectivity(retries=2, + pass_list=n1_workloads, + fail_list=n2_workloads, + type_list=['icmp', 'tcp']) + + else: + print "******************* " \ + "ERROR - Unrecognised test type " \ + "*******************" + assert False, "Unrecognised test type: %s" % test_type + + # Now restore the original profile and check it all works as before + self._apply_new_profile(original_profiles, host1) + host1.calicoctl("get profile -o yaml") + self._check_original_connectivity(n1_workloads, n2_workloads) + + # Tidy up + host1.remove_workloads() + host2.remove_workloads() + for network in networks: + network.delete() + + @staticmethod + def _get_profiles(profiles): + """ + Sorts and returns the profiles for the networks. + :param profiles: the list of profiles + :return: tuple: profile for network1, profile for network2 + """ + prof_n1 = None + prof_n2 = None + for profile in profiles: + if profile['metadata']['name'] == "testnet1": + prof_n1 = profile + elif profile['metadata']['name'] == "testnet2": + prof_n2 = profile + assert prof_n1 is not None, "Could not find testnet1 profile" + assert prof_n2 is not None, "Could not find testnet2 profile" + return prof_n1, prof_n2 + + @staticmethod + def _apply_new_profile(new_profile, host): + # Apply new profiles + host.writefile("new_profiles", + yaml.dump(new_profile, default_flow_style=False)) + host.calicoctl("apply -f new_profiles") + + def _setup_workloads(self, host1, host2): + # TODO work IPv6 into this test too + host1.start_calico_node() + host2.start_calico_node() + + # Create the networks on host1, but it should be usable from all + # hosts. We create one network using the default driver, and the + # other using the Calico driver. + network1 = host1.create_network("testnet1") + network2 = host1.create_network("testnet2") + networks = [network1, network2] + + # Assert that the networks can be seen on host2 + assert_network(host2, network2) + assert_network(host2, network1) + + n1_workloads = [] + n2_workloads = [] + + # Create two workloads on host1 and one on host2 all in network 1. + n1_workloads.append(host2.create_workload("workload_h2n1_1", + image="workload", + network=network1)) + n1_workloads.append(host1.create_workload("workload_h1n1_1", + image="workload", + network=network1)) + n1_workloads.append(host1.create_workload("workload_h1n1_2", + image="workload", + network=network1)) + + # Create similar workloads in network 2. + n2_workloads.append(host1.create_workload("workload_h1n2_1", + image="workload", + network=network2)) + n2_workloads.append(host1.create_workload("workload_h1n2_2", + image="workload", + network=network2)) + n2_workloads.append(host2.create_workload("workload_h2n2_1", + image="workload", + network=network2)) + print "*******************" + print "Network1 is:\n%s\n%s" % ( + [x.ip for x in n1_workloads], + [x.name for x in n1_workloads]) + print "Network2 is:\n%s\n%s" % ( + [x.ip for x in n2_workloads], + [x.name for x in n2_workloads]) + print "*******************" + + # Assert that endpoints are in Calico + assert_number_endpoints(host1, 4) + assert_number_endpoints(host2, 2) + + self._check_original_connectivity(n1_workloads, n2_workloads) + + # Test deleting the network. It will fail if there are any + # endpoints connected still. + self.assertRaises(CommandExecError, network1.delete) + self.assertRaises(CommandExecError, network2.delete) + + return n1_workloads, n2_workloads, networks + + def _check_original_connectivity(self, n1_workloads, n2_workloads, + types=None): + # Assert that workloads can communicate with each other on network + # 1, and not those on network 2. Ping using IP for all workloads, + # and by hostname for workloads on the same network (note that + # a workloads own hostname does not work). + if types is None: + types = ['icmp', 'tcp', 'udp'] + self.assert_connectivity(retries=2, + pass_list=n1_workloads, + fail_list=n2_workloads, + type_list=types) + + # Repeat with network 2. + self.assert_connectivity(pass_list=n2_workloads, + fail_list=n1_workloads, + type_list=types) diff --git a/tests/st/ssl-config/ca-config.json b/tests/st/ssl-config/ca-config.json new file mode 100644 index 000000000..e492de1ae --- /dev/null +++ b/tests/st/ssl-config/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ], + "expiry": "8760h" + } + } +} diff --git a/tests/st/ssl-config/ca-csr.json b/tests/st/ssl-config/ca-csr.json new file mode 100644 index 000000000..487390b45 --- /dev/null +++ b/tests/st/ssl-config/ca-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "Autogenerated CA", + "key": { + "algo": "ecdsa", + "size": 384 + }, + "names": [ + { + "O": "Certificater", + "OU": "Some other thing", + "L": "San Francisco", + "ST": "California", + "C": "US" + } + ] +} diff --git a/tests/st/ssl-config/req-csr.json b/tests/st/ssl-config/req-csr.json new file mode 100644 index 000000000..42b28dcaa --- /dev/null +++ b/tests/st/ssl-config/req-csr.json @@ -0,0 +1,19 @@ +{ + "CN": "etcd", + "hosts": [ + "localhost", + "etcd-authority-ssl", + "127.0.0.1" + ], + "key": { + "algo": "ecdsa", + "size": 384 + }, + "names": [ + { + "O": "autogenerated", + "OU": "etcd cluster", + "L": "location" + } + ] +} diff --git a/tests/st/test_base.py b/tests/st/test_base.py new file mode 100644 index 000000000..b90c3c99c --- /dev/null +++ b/tests/st/test_base.py @@ -0,0 +1,305 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import subprocess +import time +from multiprocessing.dummy import Pool as ThreadPool +from pprint import pformat +from unittest import TestCase + +import yaml +from deepdiff import DeepDiff + +from tests.st.utils.utils import (get_ip, ETCD_SCHEME, ETCD_CA, ETCD_CERT, + ETCD_KEY, debug_failures, ETCD_HOSTNAME_SSL, + wipe_etcd) + +HOST_IPV6 = get_ip(v6=True) +HOST_IPV4 = get_ip() + +logging.basicConfig(level=logging.DEBUG, format="%(message)s") +logger = logging.getLogger(__name__) + +# Disable spammy logging from the sh module +sh_logger = logging.getLogger("sh") +sh_logger.setLevel(level=logging.CRITICAL) + +first_log_time = None + + +class TestBase(TestCase): + """ + Base class for test-wide methods. + """ + + def setUp(self): + """ + Clean up before every test. + """ + self.ip = HOST_IPV4 + + self.wipe_etcd() + + # Log a newline to ensure that the first log appears on its own line. + logger.info("") + + @staticmethod + def _conn_checker(args): + source, dest, test_type, result, retries = args + if test_type == 'icmp': + if result: + return source.check_can_ping(dest, retries) + else: + return source.check_cant_ping(dest, retries) + elif test_type == 'tcp': + if result: + return source.check_can_tcp(dest, retries) + else: + return source.check_cant_tcp(dest, retries) + elif test_type == 'udp': + if result: + return source.check_can_udp(dest, retries) + else: + return source.check_cant_udp(dest, retries) + else: + logger.error("Unrecognised connectivity check test_type") + + @debug_failures + def assert_connectivity(self, pass_list, fail_list=None, retries=0, + type_list=None): + """ + Assert partial connectivity graphs between workloads. + + :param pass_list: Every workload in this list should be able to ping + every other workload in this list. + :param fail_list: Every workload in pass_list should *not* be able to + ping each workload in this list. Interconnectivity is not checked + *within* the fail_list. + :param retries: The number of retries. + :param type_list: list of types to test. If not specified, defaults to + icmp only. + """ + if type_list is None: + type_list = ['icmp', 'tcp', 'udp'] + if fail_list is None: + fail_list = [] + + conn_check_list = [] + for source in pass_list: + for dest in pass_list: + if 'icmp' in type_list: + conn_check_list.append((source, dest.ip, 'icmp', True, retries)) + if 'tcp' in type_list: + conn_check_list.append((source, dest.ip, 'tcp', True, retries)) + if 'udp' in type_list: + conn_check_list.append((source, dest.ip, 'udp', True, retries)) + for dest in fail_list: + if 'icmp' in type_list: + conn_check_list.append((source, dest.ip, 'icmp', False, retries)) + if 'tcp' in type_list: + conn_check_list.append((source, dest.ip, 'tcp', False, retries)) + if 'udp' in type_list: + conn_check_list.append((source, dest.ip, 'udp', False, retries)) + + # Empirically, 18 threads works well on my machine! + check_pool = ThreadPool(18) + results = check_pool.map(self._conn_checker, conn_check_list) + check_pool.close() + check_pool.join() + # _con_checker should only return None if there is an error in calling it + assert None not in results, ("_con_checker error - returned None") + diagstring = "" + # Check that all tests passed + if False in results: + # We've failed, lets put together some diags. + header = ["source.ip", "dest.ip", "type", "exp_result", "pass/fail"] + diagstring = "{: >18} {: >18} {: >7} {: >6} {: >6}\r\n".format(*header) + for i in range(len(conn_check_list)): + source, dest, test_type, exp_result, retries = conn_check_list[i] + pass_fail = results[i] + # Convert pass/fail into an actual result + if not pass_fail: + actual_result = not exp_result + else: + actual_result = exp_result + diag = [source.ip, dest, test_type, exp_result, actual_result] + diagline = "{: >18} {: >18} {: >7} {: >6} {: >6}\r\n".format(*diag) + diagstring += diagline + + assert False not in results, ("Connectivity check error!\r\n" + "Results:\r\n %s\r\n" % diagstring) + + @debug_failures + def assert_ip_connectivity(self, workload_list, ip_pass_list, + ip_fail_list=None, type_list=None): + """ + Assert partial connectivity graphs between workloads and given ips. + + This function is used for checking connectivity for ips that are + explicitly assigned to containers when added to calico networking. + + :param workload_list: List of workloads used to check connectivity. + :param ip_pass_list: Every workload in workload_list should be able to + ping every ip in this list. + :param ip_fail_list: Every workload in workload_list should *not* be + able to ping any ip in this list. Interconnectivity is not checked + *within* the fail_list. + :param type_list: list of types to test. If not specified, defaults to + icmp only. + """ + if type_list is None: + type_list = ['icmp'] + if ip_fail_list is None: + ip_fail_list = [] + + conn_check_list = [] + for workload in workload_list: + for ip in ip_pass_list: + if 'icmp' in type_list: + conn_check_list.append((workload, ip, 'icmp', True, 0)) + if 'tcp' in type_list: + conn_check_list.append((workload, ip, 'tcp', True, 0)) + if 'udp' in type_list: + conn_check_list.append((workload, ip, 'udp', True, 0)) + + for ip in ip_fail_list: + if 'icmp' in type_list: + conn_check_list.append((workload, ip, 'icmp', False, 0)) + if 'tcp' in type_list: + conn_check_list.append((workload, ip, 'tcp', False, 0)) + if 'udp' in type_list: + conn_check_list.append((workload, ip, 'udp', False, 0)) + + # Empirically, 18 threads works well on my machine! + check_pool = ThreadPool(18) + results = check_pool.map(self._conn_checker, conn_check_list) + check_pool.close() + check_pool.join() + # _con_checker should only return None if there is an error in calling it + assert None not in results, ("_con_checker error - returned None") + diagstring = "" + # Check that all tests passed + if False in results: + # We've failed, lets put together some diags. + header = ["source.ip", "dest.ip", "type", "exp_result", "actual_result"] + diagstring = "{: >18} {: >18} {: >7} {: >6} {: >6}\r\n".format(*header) + for i in range(len(conn_check_list)): + source, dest, test_type, exp_result, retries = conn_check_list[i] + pass_fail = results[i] + # Convert pass/fail into an actual result + if not pass_fail: + actual_result = not exp_result + else: + actual_result = exp_result + diag = [source.ip, dest, test_type, exp_result, actual_result] + diagline = "{: >18} {: >18} {: >7} {: >6} {: >6}\r\n".format(*diag) + diagstring += diagline + + assert False not in results, ("Connectivity check error!\r\n" + "Results:\r\n %s\r\n" % diagstring) + + def wipe_etcd(self): + wipe_etcd(self.ip) + + def curl_etcd(self, path, options=None, recursive=True): + """ + Perform a curl to etcd, returning JSON decoded response. + :param path: The key path to query + :param options: Additional options to include in the curl + :param recursive: Whether we want recursive query or not + :return: The JSON decoded response. + """ + curl_etcd(path, options, recursive, self.ip) + + def check_data_in_datastore(self, host, data, resource, yaml_format=True): + if yaml_format: + out = host.calicoctl( + "get %s --output=yaml" % resource) + output = yaml.safe_load(out) + else: + out = host.calicoctl( + "get %s --output=json" % resource) + output = json.loads(out) + self.assert_same(data, output) + + @staticmethod + def assert_same(thing1, thing2): + """ + Compares two things. Debug logs the differences between them before + asserting that they are the same. + """ + assert cmp(thing1, thing2) == 0, \ + "Items are not the same. Difference is:\n %s" % \ + pformat(DeepDiff(thing1, thing2), indent=2) + + @staticmethod + def writeyaml(filename, data): + """ + Converts a python dict to yaml and outputs to a file. + :param filename: filename to write + :param data: dictionary to write out as yaml + """ + with open(filename, 'w') as f: + text = yaml.dump(data, default_flow_style=False) + logger.debug("Writing %s: \n%s" % (filename, text)) + f.write(text) + + @staticmethod + def writejson(filename, data): + """ + Converts a python dict to json and outputs to a file. + :param filename: filename to write + :param data: dictionary to write out as json + """ + with open(filename, 'w') as f: + text = json.dumps(data, + sort_keys=True, + indent=2, + separators=(',', ': ')) + logger.debug("Writing %s: \n%s" % (filename, text)) + f.write(text) + + @debug_failures + def assert_false(self, b): + """ + Assert false, wrapped to allow debugging of failures. + """ + assert not b + + @debug_failures + def assert_true(self, b): + """ + Assert true, wrapped to allow debugging of failures. + """ + assert b + + @staticmethod + def log_banner(msg, *args, **kwargs): + global first_log_time + time_now = time.time() + if first_log_time is None: + first_log_time = time_now + time_now -= first_log_time + elapsed_hms = "%02d:%02d:%02d " % (time_now / 3600, + (time_now % 3600) / 60, + time_now % 60) + + level = kwargs.pop("level", logging.INFO) + msg = elapsed_hms + str(msg) % args + banner = "+" + ("-" * (len(msg) + 2)) + "+" + logger.log(level, "\n" + + banner + "\n" + "| " + msg + " |\n" + + banner) diff --git a/tests/st/utils/__init__.py b/tests/st/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/st/utils/constants.py b/tests/st/utils/constants.py new file mode 100644 index 000000000..4bc5da0af --- /dev/null +++ b/tests/st/utils/constants.py @@ -0,0 +1,20 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DEFAULT_IPV4_POOL_CIDR = "192.168.0.0/16" +DEFAULT_IPV4_ADDR_1 = "192.168.1.1" +DEFAULT_IPV4_ADDR_2 = "192.168.1.2" +DEFAULT_IPV4_ADDR_3 = "192.168.1.3" +DEFAULT_IPV6_POOL_CIDR = "fd80:24e2:f998:72d6::/64" +LARGE_AS_NUM = "4294.566" diff --git a/tests/st/utils/docker_host.py b/tests/st/utils/docker_host.py new file mode 100644 index 000000000..0f5217419 --- /dev/null +++ b/tests/st/utils/docker_host.py @@ -0,0 +1,599 @@ +# Copyright (c) 2015-2017 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import json +import os +import uuid +import yaml +from functools import partial +from subprocess import CalledProcessError, Popen, PIPE + +from log_analyzer import LogAnalyzer, FELIX_LOG_FORMAT, TIMESTAMP_FORMAT +from network import DockerNetwork +from tests.st.utils.exceptions import CommandExecError +from utils import get_ip, log_and_run, retry_until_success, ETCD_SCHEME, \ + ETCD_CA, ETCD_KEY, ETCD_CERT, ETCD_HOSTNAME_SSL +from workload import Workload + +logger = logging.getLogger(__name__) +# We want to default CHECKOUT_DIR if either the ENV var is unset +# OR its set to an empty string. +CHECKOUT_DIR = os.getenv("HOST_CHECKOUT_DIR", "") +if CHECKOUT_DIR == "": + CHECKOUT_DIR = os.getcwd() + +NODE_CONTAINER_NAME = os.getenv("NODE_CONTAINER_NAME", "calico/node:latest") + +if ETCD_SCHEME == "https": + CLUSTER_STORE_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " \ + "--cluster-store-opt kv.cacertfile=%s " \ + "--cluster-store-opt kv.certfile=%s " \ + "--cluster-store-opt kv.keyfile=%s " % \ + (ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, + ETCD_KEY) +else: + CLUSTER_STORE_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ + get_ip() + +class DockerHost(object): + """ + A host container which will hold workload containers to be networked by + Calico. + + :param calico_node_autodetect_ip: When set to True, the test framework + will not perform IP detection, and will run `calicoctl node` without + explicitly passing in a value for --ip. This means calico-node will be + forced to do its IP detection. + :param override_hostname: When set to True, the test framework will + choose an alternate hostname for the host which it will pass to all + calicoctl components as the HOSTNAME environment variable. If set + to False, the HOSTNAME environment is not explicitly set. + """ + + # A static list of Docker networks that are created by the tests. This + # list covers all Docker hosts. + docker_networks = [] + + def __init__(self, name, start_calico=True, dind=True, + additional_docker_options="", + post_docker_commands=["docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar"], + calico_node_autodetect_ip=False, + simulate_gce_routing=False, + override_hostname=False): + self.name = name + self.dind = dind + self.workloads = set() + self.ip = None + self.log_analyzer = None + """ + An IP address value to pass to calicoctl as `--ip`. If left as None, + no value will be passed, forcing calicoctl to do auto-detection. + """ + + self.ip6 = None + """ + An IPv6 address value to pass to calicoctl as `--ipv6`. If left as + None, no value will be passed. + """ + + self.override_hostname = None if not override_hostname else \ + uuid.uuid1().hex[:16] + """ + Create an arbitrary hostname if we want to override. + """ + + # This variable is used to assert on destruction that this object was + # cleaned up. If not used as a context manager, users of this object + # must invoke cleanup. + self._cleaned = False + + docker_args = "--privileged -tid " \ + "-v /lib/modules:/lib/modules " \ + "-v %s/certs:%s/certs -v %s:/code --name %s" % \ + (CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR, + self.name) + if ETCD_SCHEME == "https": + docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip()) + + if dind: + log_and_run("docker rm -f %s || true" % self.name) + # Pass the certs directory as a volume since the etcd SSL/TLS + # environment variables use the full path on the host. + # Set iptables=false to prevent iptables error when using dind + # libnetwork + log_and_run("docker run %s " + "calico/dind:latest " + "--iptables=false " + "%s" % + (docker_args, additional_docker_options)) + + self.ip = log_and_run( + "docker inspect --format " + "'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" % + self.name) + + # Make sure docker is up + docker_ps = partial(self.execute, "docker ps") + retry_until_success(docker_ps, ex_class=CalledProcessError, + retries=10) + + if simulate_gce_routing: + # Simulate addressing and routing setup as on a GCE instance: + # the instance has a /32 address (which means that it appears + # not to be directly connected to anything) and a default route + # that does not have the 'onlink' flag to override that. + # + # First check that we can ping the Docker bridge, and trace out + # initial state. + self.execute("ping -c 1 -W 2 172.17.0.1") + self.execute("ip a") + self.execute("ip r") + + # Change the normal /16 IP address to /32. + self.execute("ip a del %s/16 dev eth0" % self.ip) + self.execute("ip a add %s/32 dev eth0" % self.ip) + + # Add a default route via the Docker bridge. + self.execute("ip r a 172.17.0.1 dev eth0") + self.execute("ip r a default via 172.17.0.1 dev eth0") + + # Trace out final state, and check that we can still ping the + # Docker bridge. + self.execute("ip a") + self.execute("ip r") + self.execute("ping -c 1 -W 2 172.17.0.1") + + for command in post_docker_commands: + self.execute(command) + elif not calico_node_autodetect_ip: + # Find the IP so it can be specified as `--ip` when launching + # node later. + self.ip = get_ip(v6=False) + self.ip6 = get_ip(v6=True) + + if start_calico: + self.start_calico_node() + + def execute(self, command, raise_exception_on_failure=True): + """ + Pass a command into a host container. + + Raises a CommandExecError() if the command returns a non-zero + return code. + + :param command: The command to execute. + :return: The output from the command with leading and trailing + whitespace removed. + """ + if self.dind: + command = self.escape_shell_single_quotes(command) + command = "docker exec -it %s sh -c '%s'" % (self.name, + command) + + return log_and_run(command, raise_exception_on_failure=raise_exception_on_failure) + + def execute_readline(self, command): + """ + Execute a command and return individual lines as a generator. + Raises an exception if the return code is non-zero. Stderr is ignored. + + Use this rather than execute if the command outputs a large amount of + data that cannot be handled as a single string. + + :return: Generator of individual lines. + """ + logger.debug("Running command on %s", self.name) + logger.debug(" - Command: %s", command) + if self.dind: + command = self.escape_shell_single_quotes(command) + command = "docker exec -it %s sh -c '%s'" % (self.name, + command) + logger.debug("Final command: %s", command) + proc = Popen(command, stdout=PIPE, shell=True) + + try: + # Read and return one line at a time until no more data is + # returned. + for line in proc.stdout: + yield line + finally: + status = proc.wait() + logger.debug("- return: %s", status) + + if status: + raise Exception("Command %s returned non-zero exit code %s" % + (command, status)) + + def calicoctl(self, command, version=None): + """ + Convenience function for abstracting away calling the calicoctl + command. + + Raises a CommandExecError() if the command returns a non-zero + return code. + + :param command: The calicoctl command line parms as a single string. + :param version: The calicoctl version to use (this is appended to the + executable name. It is assumed the Makefile will ensure + the required versions are downloaded. + :return: The output from the command with leading and trailing + whitespace removed. + """ + if not version: + calicoctl = os.environ.get("CALICOCTL", "/code/dist/calicoctl") + else: + calicoctl = "/code/dist/calicoctl-" + version + + if ETCD_SCHEME == "https": + etcd_auth = "%s:2379" % ETCD_HOSTNAME_SSL + else: + etcd_auth = "%s:2379" % get_ip() + # Export the environment, in case the command has multiple parts, e.g. + # use of | or ; + # + # Pass in all etcd params, the values will be empty if not set anyway + calicoctl = "export ETCD_AUTHORITY=%s; " \ + "export ETCD_SCHEME=%s; " \ + "export ETCD_CA_CERT_FILE=%s; " \ + "export ETCD_CERT_FILE=%s; " \ + "export ETCD_KEY_FILE=%s; %s" % \ + (etcd_auth, ETCD_SCHEME, ETCD_CA, ETCD_CERT, ETCD_KEY, + calicoctl) + # If the hostname is being overriden, then export the HOSTNAME + # environment. + if self.override_hostname: + calicoctl = "export HOSTNAME=%s; %s" % ( + self.override_hostname, calicoctl) + + return self.execute(calicoctl + " " + command) + + def start_calico_node(self, options=""): + """ + Start calico in a container inside a host by calling through to the + calicoctl node command. + """ + args = ['node', 'run'] + if "--node-image" not in options: + args.append('--node-image=%s' % NODE_CONTAINER_NAME) + + # Add the IP addresses if required and we aren't explicitly specifying + # them in the options. The --ip and --ip6 options can be specified + # using "=" or space-separated parms. + if self.ip and "--ip=" not in options and "--ip " not in options: + args.append('--ip=%s' % self.ip) + if self.ip6 and "--ip6=" not in options and "--ip6 " not in options: + args.append('--ip6=%s' % self.ip6) + args.append(options) + + cmd = ' '.join(args) + self.calicoctl(cmd) + self.attach_log_analyzer() + + def set_ipip_enabled(self, enabled): + pools_output = self.calicoctl("get ippool -o yaml") + pools_dict = yaml.safe_load(pools_output) + for pool in pools_dict: + print "Pool is %s" % pool + if ':' not in pool['metadata']['cidr']: + pool['spec']['ipip'] = {'mode': 'always', 'enabled': enabled} + self.writefile("ippools.yaml", pools_dict) + self.calicoctl("apply -f ippools.yaml") + + def attach_log_analyzer(self): + self.log_analyzer = LogAnalyzer(self, + "/var/log/calico/felix/current", + FELIX_LOG_FORMAT, + TIMESTAMP_FORMAT) + + def start_calico_node_with_docker(self): + """ + Start calico in a container inside a host by calling docker directly. + """ + if ETCD_SCHEME == "https": + etcd_auth = "%s:2379" % ETCD_HOSTNAME_SSL + ssl_args = "-e ETCD_CA_CERT_FILE=%s " \ + "-e ETCD_CERT_FILE=%s " \ + "-e ETCD_KEY_FILE=%s " \ + "-v %s/certs:%s/certs " \ + % (ETCD_CA, ETCD_CERT, ETCD_KEY, + CHECKOUT_DIR, CHECKOUT_DIR) + + else: + etcd_auth = "%s:2379" % get_ip() + ssl_args = "" + + # If the hostname has been overridden on this host, then pass it in + # as an environment variable. + if self.override_hostname: + hostname_args = "-e HOSTNAME=%s" % self.override_hostname + else: + hostname_args = "" + + self.execute("docker run -d --net=host --privileged " + "--name=calico-node " + "%s " + "-e IP=%s " + "-e ETCD_AUTHORITY=%s -e ETCD_SCHEME=%s %s " + "-v /var/log/calico:/var/log/calico " + "-v /var/run/calico:/var/run/calico " + "%s" % (hostname_args, self.ip, etcd_auth, ETCD_SCHEME, + ssl_args, NODE_CONTAINER_NAME) + ) + + def remove_workloads(self): + """ + Remove all containers running on this host. + + Useful for test shut down to ensure the host is cleaned up. + :return: None + """ + for workload in self.workloads: + try: + self.execute("docker rm -f %s" % workload.name) + except CalledProcessError: + # Make best effort attempt to clean containers. Don't fail the + # test if a container can't be removed. + pass + + def remove_images(self): + """ + Remove all images running on this host. + + Useful for test shut down to ensure the host is cleaned up. + :return: None + """ + cmd = "docker rmi $(docker images -qa)" + try: + self.execute(cmd) + except CalledProcessError: + # Best effort only. + pass + + def remove_containers(self): + """ + Remove all containers running on this host. + + Useful for test shut down to ensure the host is cleaned up. + :return: None + """ + cmd = "docker rm -f $(docker ps -qa)" + try: + self.execute(cmd) + except CalledProcessError: + # Best effort only. + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exit the context of this host. + :return: None + """ + self.cleanup(log_extra_diags=bool(exc_type)) + + def cleanup(self, log_extra_diags=False): + """ + Clean up this host, including removing any containers created. This is + necessary especially for Docker-in-Docker so we don't leave dangling + volumes. + + Also, perform log analysis to check for any errors, raising an exception + if any were found. + + If log_extra_is set to True we will log some extra diagnostics (this is + set to True if the DockerHost context manager exits with an exception). + Extra logs will also be output if the log analyzer detects any errors. + """ + # Check for logs before tearing down, log extra diags if we spot an error. + log_exception = None + try: + if self.log_analyzer is not None: + self.log_analyzer.check_logs_for_exceptions() + except Exception, e: + log_exception = e + log_extra_diags = True + + # Log extra diags if we need to. + if log_extra_diags: + self.log_extra_diags() + + logger.info("# Cleaning up host %s", self.name) + if self.dind: + # For Docker-in-Docker, we need to remove all containers and + # all images. + # Start by just removing the workloads and then attempt cleanup of + # networks... + self.remove_workloads() + self.cleanup_networks() + + # ...delete any remaining containers and the images... + self.remove_containers() + self.remove_images() + + # ...and the outer container for DinD. + log_and_run("docker rm -f %s || true" % self.name) + else: + # For non Docker-in-Docker, we can only remove the containers we + # created - so remove the workloads, attempt cleanup of networks + # and delete the calico node. + self.remove_workloads() + self.cleanup_networks() + log_and_run("docker rm -f calico-node || true") + + self._cleaned = True + + # Now that tidy-up is complete, re-raise any exceptions found in the logs. + if log_exception: + raise log_exception + + def cleanup_networks(self): + """ + Attempt to cleanup any networks that are stored globally. Note that + Docker will not allow a network to be deleted whilst there are + endpoints associated with the network - thus any networks that could + not be deleted are added back to the global list and will be removed + via another docker host cleanup (after removing its endpoints). + """ + q_networks = [] + while self.docker_networks: + nw = self.docker_networks.pop() + try: + nw.delete(host=self) + except CommandExecError: + q_networks.append(nw) + self.docker_networks.extend(q_networks) + + def __del__(self): + """ + This destructor asserts this object was cleaned up before being GC'd. + + Why not just clean up? This object is used in test scripts and we + can't guarantee that GC will happen between test runs. So, un-cleaned + objects may result in confusing behaviour since this object manipulates + Docker containers running on the system. + :return: + """ + assert self._cleaned + + def create_workload(self, name, image="busybox", network="bridge", ip=None, labels=[]): + """ + Create a workload container inside this host container. + """ + workload = Workload(self, name, image=image, network=network, ip=ip, labels=labels) + self.workloads.add(workload) + return workload + + def create_network(self, name, driver="calico", ipam_driver="calico-ipam", + subnet=None): + """ + Create a Docker network using this host. If the DockerHost is used + as a context manager, exit processing will attempt deletion of *all* + networks created across *all* Docker hosts - if you do not want the + tidy up of networks to occur automatically, don't use the DockerHost as + a context manager and perform tidy explicitly. + + :param name: The name of the network. This must be unique per cluster + and it is the user-facing identifier for the network. + :param driver: The name of the network driver to use. (The Calico + driver is the default.) + :param ipam_driver: The name of the IPAM driver to use. (The Calico + driver is the default.) + :param subnet: The subnet IP pool to assign IPs from. + :return: A DockerNetwork object. + """ + nw = DockerNetwork(self, name, driver=driver, ipam_driver=ipam_driver, + subnet=subnet) + + # Store the network so that we can attempt to remove it when this host + # or another host exits. + self.docker_networks.append(nw) + return nw + + @staticmethod + def escape_shell_single_quotes(command): + """ + Escape single quotes in shell strings. + + Replace ' (single-quote) in the command with an escaped version. + This needs to be done, since the command is passed to "docker + exec" to execute and needs to be single quoted. + Strictly speaking, it's impossible to escape single-quoted + shell script, but there is a workaround - end the single quoted + string, then concatenate a double quoted single quote, + and finally re-open the string with a single quote. Because + this is inside a single quoted python, string, the single + quotes also need escaping. + + :param command: The string to escape. + :return: The escaped string + """ + return command.replace('\'', '\'"\'"\'') + + def get_hostname(self): + """ + Get the hostname from Docker + The hostname is a randomly generated string. + Note, this function only works with a host with dind enabled. + Raises an exception if dind is not enabled. + + :return: hostname of DockerHost + """ + # If overriding the hostname, return that one. + if self.override_hostname: + return self.override_hostname + + command = "docker inspect --format {{.Config.Hostname}} %s" % self.name + return log_and_run(command) + + def writefile(self, filename, data): + """ + Writes a file on a host (e.g. a yaml file for loading into calicoctl). + :param filename: string, the filename to create + :param data: string, the data to put inthe file + :return: Return code of execute operation. + """ + return self.execute("cat << EOF > %s\n%s" % (filename, data)) + + def writejson(self, filename, data): + """ + Converts a python dict to json and outputs to a file. + :param filename: filename to write + :param data: dictionary to write out as json + """ + text = json.dumps(data, + sort_keys=True, + indent=2, + separators=(',', ': ')) + self.writefile(filename, text) + + def add_resource(self, resource_data): + """ + Add resource specified in resource_data object. + :param resource_data: object representing json data for the resource + to add + """ + self._apply_resources(resource_data) + + def delete_all_resource(self, resource): + """ + Delete all resources of the specified type. + :param resource: string, resource type to delete + """ + # Grab all objects of a resource type + objects = yaml.load(self.calicoctl("get %s -o yaml" % resource)) + # and delete them (if there are any) + if len(objects) > 0: + self._delete_data(objects) + + def _delete_data(self, data): + logger.debug("Deleting data with calicoctl: %s", data) + self._exec_calicoctl("delete", data) + + def _apply_resources(self, resources): + self._exec_calicoctl("apply", resources) + + def _exec_calicoctl(self, action, data): + # use calicoctl with data + self.writejson("new_data", data) + self.calicoctl("%s -f new_data" % action) + + def log_extra_diags(self): + # Run a set of commands to trace ip routes, iptables and ipsets. + self.execute("ip route", raise_exception_on_failure=False) + self.execute("iptables-save", raise_exception_on_failure=False) + self.execute("ip6tables-save", raise_exception_on_failure=False) + self.execute("ipset save", raise_exception_on_failure=False) diff --git a/tests/st/utils/exceptions.py b/tests/st/utils/exceptions.py new file mode 100644 index 000000000..45b278755 --- /dev/null +++ b/tests/st/utils/exceptions.py @@ -0,0 +1,42 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from subprocess import CalledProcessError + + +class CommandExecError(CalledProcessError): + """ + Wrapper for CalledProcessError with an Exception message that gives the + output captured from the failed command. + """ + + def __init__(self, called_process_error): + self.called_process_error = called_process_error + + @property + def returncode(self): + return self.called_process_error.returncode + + @property + def output(self): + return self.called_process_error.output + + @property + def cmd(self): + return self.called_process_error.cmd + + def __str__(self): + return "Command %s failed with RC %s and output:\n%s" % \ + (self.called_process_error.cmd, + self.called_process_error.returncode, + self.called_process_error.output) diff --git a/tests/st/utils/log_analyzer.py b/tests/st/utils/log_analyzer.py new file mode 100644 index 000000000..3fbc59472 --- /dev/null +++ b/tests/st/utils/log_analyzer.py @@ -0,0 +1,355 @@ +# Copyright (c) 2015-2017 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.ra, Inc. All rights reserved. + +from collections import deque +from datetime import datetime +import logging +import re + +from tests.st.utils.exceptions import CommandExecError + +_log = logging.getLogger(__name__) + +FELIX_LOG_FORMAT = ( + "(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).\d{0,3} " + "\\[(?P\w+)\\]" + "\\[(?P\d+)(/\d+)?\\] " + "(?P.*)" +) + +TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S" + +# The number of additional logs to trace out before the first error log, and +# the maximum number of errors to report. +NUM_CONTEXT_LOGS = 300 +MAX_NUM_ERRORS = 100 + +# This is the list of logs we should ignore for all tests. +LOGS_IGNORE_ALL_TESTS = [ + "Failed to connect to syslog error=Unix syslog delivery error level=", + "Exiting. reason=\"config changed\"", +] + + +class Log(object): + """ + Class encapsulating information about a log extracted from a log file. + """ + + def __init__(self, timestamp, level, pid, msg): + """ + :param timestamp: The log datetime. + :param level: The log level + :param pid: The PID of the process that created the log + :param msg: The log text. + """ + self.timestamp = timestamp + self.level = level + self.pid = pid + self.msg = msg + + def append(self, logtext): + """ + Append the text to the end of the current text with a newline + separator. + + :param logtext: The text to append to the log. + """ + self.msg += "\n" + logtext + + def detailed(self): + return "=== LOG %s %s [pid %s] ===\n%s" % (self.timestamp, self.level, + self.pid, self.msg) + + def __str__(self): + return "%s %s %s %s" % (self.timestamp, self.level, + self.pid, self.msg) + + def __repr__(self): + return self.__str__() + +class LogAnalyzer(object): + """ + LogAnalyzer class to check any new logs generated since the analyzer + was instantiated. + + This is a fairly simpler parser - it doesn't check flipped files. + """ + + def __init__(self, host, filename, log_format, timestamp_format, + continuation_level=None): + """ + :param host: the host running calico-node + :param filename: The log filename on the server + :param log_format: The format of the logs + :param timestamp_format: The date/time format in the logs. + :param continuation_level: An optional log level that indicates the + log is a continuation of the previous log (i.e. the text can be + extracted and appended to the previous log). + + The log format should be a regex string containing the following + named matches: + - timestamp (the extracted timestamp) + - loglevel (the log level) + - pid (the process ID) + - logtext (the actual log message) + + The timestamp format is the format of the extracted timestamp in + notation used by datetime.datetime.strptime(). + """ + self.host = host + self.filename = filename + self.log_regex = re.compile(log_format) + self.timestamp_format = timestamp_format + self.init_log_time = None + self.init_log_lines = None + self.continuation_level = continuation_level + + # Store the time of the last log in the file. + self.reset() + + def reset(self): + """ + Initialise the time of the first log in the log file and the number + of lines in the log file. + + This information is used to work out where to start from when looking + at new logs. + """ + _log.debug("Resetting log analyzer on %s", self.host.name) + # Grab the time of the first log. + self.init_log_time = self._get_first_log_time() + _log.debug("First log has timestamp: %s", self.init_log_time) + + self.init_log_lines = self._get_logs_num_lines() + _log.debug("Log file has %s lines", self.init_log_lines) + + def _get_first_log_time(self): + """ + Extract the time of the first log in the file. This is used to + determine whether a file has flipped during a test. + """ + cmd = "head -100 %s" % self.filename + for log in self._parse_logs(cmd): + return log.timestamp + return None + + def _get_logs_num_lines(self): + """ + Return the number of lines in the log file. + + :return: The number of lines in the log file or None if the file does + not exist or cannot be read. + """ + cmd = "wc -l %s" % self.filename + lines = None + stdout = None + try: + stdout = self.host.execute(cmd) + except CommandExecError: + _log.debug("Error running command: %s", cmd) + + _log.debug("Extract number of lines in file: %s", + self.filename) + try: + lines = int(stdout.split(" ")[0]) + except ValueError: + _log.error("Unable to parse output: %s", stdout) + except AttributeError: + _log.error("None output?: %s", stdout) + + return lines + + def get_latest_logs(self, logfilter=None): + """ + Get the latest (filtered) logs from the server. + + :param logfilter: An optional filter that determines whether a log + should be stored. This is a function that takes the log as the only + argument and returns True if the log should be filtered _out_ of the + list. + :return: A list of Log objects. + """ + return [log for log in self._parse_latest_logs() if not logfilter or not logfilter(log)] + + def _parse_latest_logs(self): + """ + Parse the latest logs from the server, returning a generator that + iterates through the logs. + + :return: A Log generator. + """ + # Use the entire log file if the file has flipped (i.e. the first log + # time is not the same, otherwise tail all but the first logs. + first_log_time = self._get_first_log_time() + _log.debug("First log has timestamp: %s", first_log_time) + + if first_log_time != self.init_log_time or \ + not self.init_log_lines: + _log.debug("Log file is new") + cmd = "cat %s" % self.filename + else: + _log.debug("Check appended logs") + cmd = "tail -n +%s %s" % (self.init_log_lines + 1, + self.filename) + return self._parse_logs(cmd) + + def _parse_logs(self, cmd): + """ + Parse the logs from the output of the supplied command, returning a + generator that iterates through the logs. + + :param cmd: The command to run to output the logs. + + :return: A Log generator. + """ + last_log = None + try: + for line in self.host.execute_readline(cmd): + log = self._process_log_line(line, last_log) + + # Logs may be continued, in which case we only return the log + # when the parsing indicates a new log. + if last_log and last_log != log: + yield last_log + last_log = log + except Exception: + _log.exception( + "Hit exception getting logs from %s - skip logs", + self.host.name) + + # Yield the final log. + if last_log: + yield last_log + + def _process_log_line(self, line, last_log): + """ + Build up a list of logs from the supplied log line. + + If a line in the logs_text does not match the format of the log string + it is assumed it is a continuation of the previous log. Similarly, + a log with level "TRACE" is also treated as a continuation. + + :param line: The log line to process. This may either add a new log + or may be a continuation of a previous log, or may be filtered out. + :param last_log: The previous log that was processed by this command. + This may be None for the first line in the log file. + :return: The log that was added or updated by this method. This may + return None if no log was parsed. If this line was appended to the + previous log, it will return last_log. + """ + # Put the full text of the log into logtext, but strip off ending whitespace because + # we'll add \n back to it when we append to it + logtext = line.rstrip() + # Strip superfluous whitespace + line = line.strip() + + # Check the line for a log match. + log_match = self.log_regex.match(line) + + # If the line does not match the regex it will be a continuation + # of the previous log. If there was no previous log then we must + # have starting parsing in the middle of a multi-line log. + if not log_match: + if last_log: + last_log.append(line) + return last_log + + # Extract the parameters from the match object. + groupdict = log_match.groupdict() + loglevel = groupdict["loglevel"] + timestamp = datetime.strptime(groupdict["timestamp"], + self.timestamp_format) + pid = groupdict["pid"] + + # Neutron logs use a log level of TRACE to continue a multi-line + # log. If there was no previous log then we must have starting parsing + # in the middle of a multi-line log. + if self.continuation_level == loglevel: + if last_log: + last_log.append(logtext) + return last_log + + # Create and return the new log. We don't add it until we start the + # next log as we need to get the entire log before we can run it + # through the filter. + log = Log(timestamp, loglevel, pid, logtext) + return log + + def check_logs_for_exceptions(self): + """ + Check the logs for any error level logs and raises an exception if + any are found. + """ + _log.info("Checking logs for exceptions") + _log.debug("Analyzing logs from %s on %s", + self.filename, self.host.name) + + # Store each error with a set of preceeding context logs. + errors = [] + logs = deque(maxlen=NUM_CONTEXT_LOGS) + + # Iterate through the logs finding all error logs and keeping track + # of unfiltered context logs. + for log in self._parse_latest_logs(): + logs.append(log) + if self._is_error_log(log): + errors.append(logs) + logs = deque(maxlen=NUM_CONTEXT_LOGS) + + # Limit the number of errors we report. + if len(errors) == MAX_NUM_ERRORS: + break + + if errors: + # Trace out the error logs (this is the last entry in each of the + # error deques). + _log.error("***** Start of errors in logs from %s on %s *****" + "\n\n%s\n\n", + self.filename, self.host.name, + "\n\n".join(map(lambda logs: logs[-1].detailed(), errors))) + _log.error("****** End of errors in logs from %s on %s ******", + self.filename, self.host.name) + + if len(errors) == MAX_NUM_ERRORS: + _log.error("Limited to %d errors reported" % MAX_NUM_ERRORS) + + # Trace out the unfiltered logs - each error stored above contains a set + # proceeding context logs followed by the error log. Join them all + # together to trace out, delimiting groups of logs with a "..." to + # indicate that some logs in between may be missing (because we only + # trace out a max number of proceeding logs). + _log.error("***** Start of context logs from %s on %s *****" + "\n\n%s\n\n", + self.filename, self.host.name, + "\n...\n".join(map(lambda logs: "\n".join(map(str, logs)), errors))) + _log.error("****** End of context logs from %s on %s ******", + self.filename, self.host.name) + + assert not errors, "Test suite failed due to errors raised in logs" + + def _is_error_log(self, log): + """ + Return whether the log is an error log or not. + + :return: True if the log is an error log. + + Note that we are skipping known failures as defined by the + LOGS_IGNORE_ALL_TESTS. + """ + is_error = log.level in {"ERROR", "PANIC", "FATAL", "CRITICAL"} + if is_error: + is_error = not any(txt in log.msg for txt in LOGS_IGNORE_ALL_TESTS) + + return is_error diff --git a/tests/st/utils/network.py b/tests/st/utils/network.py new file mode 100644 index 000000000..66a3d44f8 --- /dev/null +++ b/tests/st/utils/network.py @@ -0,0 +1,93 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from functools import partial +from tests.st.utils.exceptions import CommandExecError +from tests.st.utils.utils import retry_until_success + +logger = logging.getLogger(__name__) + +class DockerNetwork(object): + """ + A Docker network created by libnetwork. + + Docker networks provide mutual connectivity to the endpoints attached to + them (and endpoints join/leave sandboxes which are network namespaces used + by containers). + """ + + def __init__(self, host, name, driver="calico", ipam_driver="calico-ipam", + subnet=None): + """ + Create the network. + :param host: The Docker Host which creates the network + :param name: The name of the network. This must be unique per cluster + and is the user-facing identifier for the network. (Calico itself will + get a UUID for the network via the driver API and will not get the + name). + :param driver: The name of the network driver to use. (The Calico + driver is the default.) + :param ipam_driver: The name of the IPAM driver to use, or None to use + the default driver. + :param subnet: The subnet IP pool to assign IPs from. + :return: A DockerNetwork object. + """ + self.name = name + self.driver = driver + self.deleted = False + + self.init_host = host + """The host which created the network.""" + + driver_option = ("--driver %s" % driver) if driver else "" + ipam_option = ("--ipam-driver %s" % ipam_driver) if ipam_driver else "" + subnet_option = ("--subnet %s" % subnet) if subnet else "" + + # Check if network is present before we create it + try: + host.execute("docker network inspect %s" % name) + # Network exists - delete it + host.execute("docker network rm " + name) + except CommandExecError: + # Network didn't exist, no problem. + pass + + # Create the network, + cmd = "docker network create %s %s %s %s" % \ + (driver_option, ipam_option, subnet_option, name) + docker_net_create = partial(host.execute, cmd) + self.uuid = retry_until_success(docker_net_create) + + def delete(self, host=None): + """ + Delete the network. + :param host: The Docker Host to use when deleting the network. If + not specified, defaults to the host used to create the network. + :return: Nothing + """ + if not self.deleted: + host = host or self.init_host + host.execute("docker network rm " + self.name) + self.deleted = True + + def disconnect(self, host, container): + """ + Disconnect container from network. + :return: Nothing + """ + host.execute("docker network disconnect %s %s" % + (self.name, str(container))) + + def __str__(self): + return self.name diff --git a/tests/st/utils/route_reflector.py b/tests/st/utils/route_reflector.py new file mode 100644 index 000000000..8d6f5f256 --- /dev/null +++ b/tests/st/utils/route_reflector.py @@ -0,0 +1,120 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from docker_host import DockerHost, CHECKOUT_DIR +from utils import get_ip, ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL + +from netaddr import IPAddress + +class RouteReflectorCluster(object): + """ + Encapsulate the setting up and tearing down of a route reflector cluster. + """ + + def __init__(self, num_in_redundancy_group, num_redundancy_groups): + """ + :param num_rrs: The number of route reflectors in the cluster. + """ + self.num_in_redundancy_group = num_in_redundancy_group + self.num_redundancy_groups = num_redundancy_groups + self.redundancy_groups = [] + + def __enter__(self): + """ + Set up the route reflector clusters when entering context. + :return: self. + """ + # Create the route reflector hosts, grouped by redundancy. + for ii in range(self.num_redundancy_groups): + cluster_id = str(IPAddress(0xFF000001 + ii)) + redundancy_group = [] + for jj in range(self.num_in_redundancy_group): + rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False) + ip_env = "-e IP=%s" % rr.ip + rr.execute("docker load --input /code/routereflector.tar") + + # Check which type of etcd is being run, then invoke the + # suggested curl command to add the RR entry to etcd. + # + # See https://github.com/projectcalico/calico-bird/tree/feature-ipinip/build_routereflector + # for details. + if os.getenv("ETCD_SCHEME", None) == "https": + # Etcd is running with SSL/TLS, pass the key values + rr.execute("docker run --privileged --net=host -d " + "--name rr %s " + "-e ETCD_ENDPOINTS=https://%s:2379 " + "-e ETCD_CA_CERT_FILE=%s " + "-e ETCD_CERT_FILE=%s " + "-e ETCD_KEY_FILE=%s " + "-v %s/certs:%s/certs " + "calico/routereflector" % + (ip_env, ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, + ETCD_KEY, CHECKOUT_DIR,CHECKOUT_DIR)) + rr.execute(r'curl --cacert %s --cert %s --key %s ' + r'-L https://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s ' + r'-XPUT -d value="{' + r'\"ip\":\"%s\",' + r'\"cluster_id\":\"%s\"' + r'}"' % (ETCD_CA, ETCD_CERT, ETCD_KEY, + ETCD_HOSTNAME_SSL, rr.ip, rr.ip, + cluster_id)) + + else: + rr.execute("docker run --privileged --net=host -d " + "--name rr %s " + "-e ETCD_ENDPOINTS=http://%s:2379 " + "calico/routereflector" % (ip_env, get_ip())) + rr.execute(r'curl -L http://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s ' + r'-XPUT -d value="{' + r'\"ip\":\"%s\",' + r'\"cluster_id\":\"%s\"' + r'}"' % (get_ip(), rr.ip, rr.ip, cluster_id)) + # Store the redundancy group. + redundancy_group.append(rr) + self.redundancy_groups.append(redundancy_group) + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Tear down the route reflector hosts when exiting context. + :return: None + """ + # Try to clean up what we can before exiting. + for rg in self.redundancy_groups: + while rg: + try: + self.pop_and_cleanup_route_reflector(rg) + except KeyboardInterrupt: + raise + except Exception: + pass + + def pop_and_cleanup_route_reflector(self, redundancy_group): + """ + Pop a route reflector off the stack and clean it up. + """ + rr = redundancy_group.pop() + rr.cleanup() + + def get_redundancy_group(self): + """ + Return a redundancy group to use. This iterates through redundancy + groups each invocation. + :return: A list of RRs in the redundancy group. + """ + rg = self.redundancy_groups.pop(0) + self.redundancy_groups.append(rg) + return rg diff --git a/tests/st/utils/utils.py b/tests/st/utils/utils.py new file mode 100644 index 000000000..9388397f9 --- /dev/null +++ b/tests/st/utils/utils.py @@ -0,0 +1,387 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os +import pdb +import re +import socket +import sys +from subprocess import CalledProcessError +from subprocess import check_output, STDOUT +from time import sleep + +import termios +import yaml +from netaddr import IPNetwork, IPAddress +from exceptions import CommandExecError + +LOCAL_IP_ENV = "MY_IP" +LOCAL_IPv6_ENV = "MY_IPv6" +logger = logging.getLogger(__name__) + +ETCD_SCHEME = os.environ.get("ETCD_SCHEME", "http") +ETCD_CA = os.environ.get("ETCD_CA_CERT_FILE", "") +ETCD_CERT = os.environ.get("ETCD_CERT_FILE", "") +ETCD_KEY = os.environ.get("ETCD_KEY_FILE", "") +ETCD_HOSTNAME_SSL = "etcd-authority-ssl" + +""" +Compile Regexes +""" +# Splits into groups that start w/ no whitespace and contain all lines below +# that start w/ whitespace +INTERFACE_SPLIT_RE = re.compile(r'(\d+:.*(?:\n\s+.*)+)') +# Grabs interface name +IFACE_RE = re.compile(r'^\d+: (\S+):') +# Grabs v4 addresses +IPV4_RE = re.compile(r'inet ((?:\d+\.){3}\d+)/\d+') +# Grabs v6 addresses +IPV6_RE = re.compile(r'inet6 ([a-fA-F\d:]+)/\d{1,3}') + + +def get_ip(v6=False): + """ + Return a string of the IP of the hosts interface. + Try to get the local IP from the environment variables. This allows + testers to specify the IP address in cases where there is more than one + configured IP address for the test system. + """ + env = LOCAL_IPv6_ENV if v6 else LOCAL_IP_ENV + ip = os.environ.get(env) + if not ip: + try: + logger.debug("%s not set; try to auto detect IP.", env) + socket_type = socket.AF_INET6 if v6 else socket.AF_INET + s = socket.socket(socket_type, socket.SOCK_DGRAM) + remote_ip = "2001:4860:4860::8888" if v6 else "8.8.8.8" + s.connect((remote_ip, 0)) + ip = s.getsockname()[0] + s.close() + except BaseException: + # Failed to connect, just try to get the address from the interfaces + version = 6 if v6 else 4 + ips = get_host_ips(version) + if ips: + ip = str(ips[0]) + else: + logger.debug("Got local IP from %s=%s", env, ip) + + return ip + + +# Some of the commands we execute like to mess with the TTY configuration, which can break the +# output formatting. As a wrokaround, save off the terminal settings and restore them after +# each command. +_term_settings = termios.tcgetattr(sys.stdin.fileno()) + + +def log_and_run(command, raise_exception_on_failure=True): + def log_output(results): + if results is None: + logger.info(" # ") + + lines = results.split("\n") + for line in lines: + logger.info(" # %s", line.rstrip()) + + try: + logger.info("%s", command) + try: + results = check_output(command, shell=True, stderr=STDOUT).rstrip() + finally: + # Restore terminal settings in case the command we ran manipulated them. Note: + # under concurrent access, this is still not a perfect solution since another thread's + # child process may break the settings again before we log below. + termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, _term_settings) + log_output(results) + return results + except CalledProcessError as e: + # Wrap the original exception with one that gives a better error + # message (including command output). + logger.info(" # Return code: %s", e.returncode) + log_output(e.output) + if raise_exception_on_failure: + raise CommandExecError(e) + + +def retry_until_success(function, retries=10, ex_class=Exception): + """ + Retries function until no exception is thrown. If exception continues, + it is reraised. + + :param function: the function to be repeatedly called + :param retries: the maximum number of times to retry the function. + A value of 0 will run the function once with no retries. + :param ex_class: The class of expected exceptions. + :returns: the value returned by function + """ + for retry in range(retries + 1): + try: + result = function() + except ex_class: + if retry < retries: + sleep(1) + else: + raise + else: + # Successfully ran the function + return result + + +def debug_failures(fn): + """ + Decorator function to decorate assertion methods to pause the live system + when an assertion fails, allowing the user to debug the problem. + :param fn: The function to decorate. + :return: The decorated function. + """ + + def wrapped(*args, **kwargs): + try: + return fn(*args, **kwargs) + except KeyboardInterrupt: + raise + except Exception as e: + if (os.getenv("DEBUG_FAILURES") is not None and + os.getenv("DEBUG_FAILURES").lower() == "true"): + logger.error("TEST FAILED:\n%s\nEntering DEBUG mode." + % e.message) + pdb.set_trace() + else: + raise + + return wrapped + + +@debug_failures +def check_bird_status(host, expected): + """ + Check the BIRD status on a particular host to see if it contains the + expected BGP status. + + :param host: The host object to check. + :param expected: A list of tuples containing: + (peertype, ip address, state) + where 'peertype' is one of "Global", "Mesh", "Node", 'ip address' is + the IP address of the peer, and state is the expected BGP state (e.g. + "Established" or "Idle"). + """ + output = host.calicoctl("node status") + lines = output.split("\n") + for (peertype, ipaddr, state) in expected: + for line in lines: + # Status table format is of the form: + # +--------------+-------------------+-------+----------+-------------+ + # | Peer address | Peer type | State | Since | Info | + # +--------------+-------------------+-------+----------+-------------+ + # | 172.17.42.21 | node-to-node mesh | up | 16:17:25 | Established | + # | 10.20.30.40 | global | start | 16:28:38 | Connect | + # | 192.10.0.0 | node specific | start | 16:28:57 | Connect | + # +--------------+-------------------+-------+----------+-------------+ + # + # Splitting based on | separators results in an array of the + # form: + # ['', 'Peer address', 'Peer type', 'State', 'Since', 'Info', ''] + columns = re.split("\s*\|\s*", line.strip()) + if len(columns) != 7: + continue + + if type(state) is not list: + state = [state] + + # Find the entry matching this peer. + if columns[1] == ipaddr and columns[2] == peertype: + + # Check that the connection state is as expected. We check + # that the state starts with the expected value since there + # may be additional diagnostic information included in the + # info field. + if any(columns[5].startswith(s) for s in state): + break + else: + msg = "Error in BIRD status for peer %s:\n" \ + "Expected: %s; Actual: %s\n" \ + "Output:\n%s" % (ipaddr, state, columns[5], + output) + raise AssertionError(msg) + else: + msg = "Error in BIRD status for peer %s:\n" \ + "Type: %s\n" \ + "Expected: %s\n" \ + "Output: \n%s" % (ipaddr, peertype, state, output) + raise AssertionError(msg) + + +@debug_failures +def assert_number_endpoints(host, expected): + """ + Check that a host has the expected number of endpoints in Calico + Parses the "calicoctl endpoint show" command for number of endpoints. + Raises AssertionError if the number of endpoints does not match the + expected value. + + :param host: DockerHost object + :param expected: int, number of expected endpoints + :return: None + """ + hostname = host.get_hostname() + out = host.calicoctl("get workloadEndpoint -o yaml") + output = yaml.safe_load(out) + actual = 0 + for endpoint in output: + if endpoint['metadata']['node'] == hostname: + actual += 1 + + if int(actual) != int(expected): + raise AssertionError( + "Incorrect number of endpoints on host %s: \n" + "Expected: %s; Actual: %s" % (hostname, expected, actual) + ) + + +@debug_failures +def assert_profile(host, profile_name): + """ + Check that profile is registered in Calico + Parse "calicoctl profile show" for the given profilename + + :param host: DockerHost object + :param profile_name: String of the name of the profile + :return: Boolean: True if found, False if not found + """ + out = host.calicoctl("get -o yaml profile") + output = yaml.safe_load(out) + found = False + for profile in output: + if profile['metadata']['name'] == profile_name: + found = True + break + + if not found: + raise AssertionError("Profile %s not found in Calico" % profile_name) + + +def get_profile_name(host, network): + """ + Get the profile name from Docker + A profile is created in Docker for each Network object. + The profile name is a randomly generated string. + + :param host: DockerHost object + :param network: Network object + :return: String: profile name + """ + info_raw = host.execute("docker network inspect %s" % network.name) + info = json.loads(info_raw) + + # Network inspect returns a list of dicts for each network being inspected. + # We are only inspecting 1, so use the first entry. + return info[0]["Id"] + + +@debug_failures +def assert_network(host, network): + """ + Checks that the given network is in Docker + Raises an exception if the network is not found + + :param host: DockerHost object + :param network: Network object + :return: None + """ + try: + host.execute("docker network inspect %s" % network.name) + except CommandExecError: + raise AssertionError("Docker network %s not found" % network.name) + + +@debug_failures +def get_host_ips(version=4, exclude=None): + """ + Gets all IP addresses assigned to this host. + + Ignores Loopback Addresses + + This function is fail-safe and will return an empty array instead of + raising any exceptions. + + :param version: Desired IP address version. Can be 4 or 6. defaults to 4 + :param exclude: list of interface name regular expressions to ignore + (ex. ["^lo$","docker0.*"]) + :return: List of IPAddress objects. + """ + exclude = exclude or [] + ip_addrs = [] + + # Select Regex for IPv6 or IPv4. + ip_re = IPV4_RE if version is 4 else IPV6_RE + + # Call `ip addr`. + try: + ip_addr_output = check_output(["ip", "-%d" % version, "addr"]) + except (CalledProcessError, OSError): + print("Call to 'ip addr' Failed") + sys.exit(1) + + # Separate interface blocks from ip addr output and iterate. + for iface_block in INTERFACE_SPLIT_RE.findall(ip_addr_output): + # Try to get the interface name from the block + match = IFACE_RE.match(iface_block) + iface = match.group(1) + # Ignore the interface if it is explicitly excluded + if match and not any(re.match(regex, iface) for regex in exclude): + # Iterate through Addresses on interface. + for address in ip_re.findall(iface_block): + # Append non-loopback addresses. + if not IPNetwork(address).ip.is_loopback(): + ip_addrs.append(IPAddress(address)) + + return ip_addrs + +def curl_etcd(path, options=None, recursive=True, ip=None): + """ + Perform a curl to etcd, returning JSON decoded response. + :param path: The key path to query + :param options: Additional options to include in the curl + :param recursive: Whether we want recursive query or not + :return: The JSON decoded response. + """ + if options is None: + options = [] + if ETCD_SCHEME == "https": + # Etcd is running with SSL/TLS, require key/certificates + rc = check_output( + "curl --cacert %s --cert %s --key %s " + "-sL https://%s:2379/v2/keys/%s?recursive=%s %s" + % (ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL, + path, str(recursive).lower(), " ".join(options)), + shell=True) + else: + rc = check_output( + "curl -sL http://%s:2379/v2/keys/%s?recursive=%s %s" + % (ip, path, str(recursive).lower(), " ".join(options)), + shell=True) + + return json.loads(rc.strip()) + +def wipe_etcd(ip): + # Delete /calico if it exists. This ensures each test has an empty data + # store at start of day. + curl_etcd("calico", options=["-XDELETE"], ip=ip) + + # Disable Usage Reporting to usage.projectcalico.org + # We want to avoid polluting analytics data with unit test noise + curl_etcd("calico/v1/config/UsageReportingEnabled", + options=["-XPUT -d value=False"], ip=ip) diff --git a/tests/st/utils/workload.py b/tests/st/utils/workload.py new file mode 100644 index 000000000..243998531 --- /dev/null +++ b/tests/st/utils/workload.py @@ -0,0 +1,360 @@ +# Copyright (c) 2015-2016 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from functools import partial + +from netaddr import IPAddress + +from exceptions import CommandExecError +from utils import retry_until_success, debug_failures + +NET_NONE = "none" + +logger = logging.getLogger(__name__) + + +class Workload(object): + """ + A calico workload. + + These are the end-users containers that will run application-level + software. + """ + + def __init__(self, host, name, image="busybox", network="bridge", ip=None, labels=[]): + """ + Create the workload and detect its IPs. + + :param host: The host container on which this workload is instantiated. + All commands executed by this container will be passed through the host + via docker exec. + :param name: The name given to the workload container. This name is + passed to docker and can be used inside docker commands. + :param image: The docker image to be used to instantiate this + container. busybox used by default because it is extremely small and + has ping. + :param network: The DockerNetwork to connect to. Set to None to use + default Docker networking. + :param ip: The ip address to assign to the container. + :param labels: List of labels '=' to add to workload. + """ + self.host = host + self.name = name + + lbl_args = "" + for label in labels: + lbl_args += " --label %s" % (label) + + ip_option = ("--ip %s" % ip) if ip else "" + command = "docker run -tid --name %s --net %s %s %s %s" % \ + (name, network, lbl_args, ip_option, image) + docker_run_wl = partial(host.execute, command) + retry_until_success(docker_run_wl) + self.ip = host.execute( + "docker inspect --format " + "'{{.NetworkSettings.Networks.%s.IPAddress}}' %s" + % (network, name)) + + def execute(self, command): + """ + Execute arbitrary commands on this workload. + """ + # Make sure we've been created in the context of a host. Done here + # instead of in __init__ as we can't exist in the host until we're + # created. + assert self in self.host.workloads + return self.host.execute("docker exec %s %s" % (self.name, command)) + + def _get_ping_function(self, ip): + """ + Return a function to ping the supplied IP address from this workload. + + :param ip: The IPAddress to ping. + :return: A partial function that can be executed to perform the ping. + The function raises a CommandExecError exception if the ping fails, + or returns the output of the ping. + """ + # Default to "ping" + ping = "ping" + + try: + version = IPAddress(ip).version + assert version in [4, 6] + if version == 6: + ping = "ping6" + except BaseException: + pass + + args = [ + ping, + "-c", "1", # Number of pings + "-W", "1", # Timeout for each ping + ip, + ] + command = ' '.join(args) + + ping = partial(self.execute, command) + return ping + + @debug_failures + def check_can_ping(self, ip, retries=0): + """ + Execute a ping from this workload to the ip. Assert than a workload + can ping an IP. Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently unreachable. + + :param ip: The IP address (str or IPAddress) to ping. + :param retries: The number of retries. + :return: None. + """ + try: + retry_until_success(self._get_ping_function(ip), + retries=retries, + ex_class=CommandExecError) + except CommandExecError: + return False + + return True + + @debug_failures + def check_cant_ping(self, ip, retries=0): + """ + Execute a ping from this workload to the ip. Assert that the workload + cannot ping an IP. Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently reachable. + + :param ip: The IP address (str or IPAddress) to ping. + :param retries: The number of retries. + :return: None. + """ + ping = self._get_ping_function(ip) + + def cant_ping(): + try: + ping() + except CommandExecError: + pass + else: + raise _PingError() + + try: + retry_until_success(cant_ping, + retries=retries, + ex_class=_PingError) + except _PingError: + return False + + return True + + def _get_tcp_function(self, ip): + """ + Return a function to check tcp connectivity to another ip. + + :param ip: The ip to check against. + :return: A partial function that can be executed to perform the check. + The function raises a CommandExecError exception if the check fails, + or returns the output of the check. + """ + # test_string = "hello" + args = [ + "/code/tcpping.sh", + ip, + ] + + command = ' '.join(args) + + tcp_check = partial(self.execute, command) + return tcp_check + + def _get_tcp_asym_function(self, ip): + """ + Return a function to check tcp connectivity to another ip. + + :param ip: The ip to check against. + :return: A partial function that can be executed to perform the check. + The function raises a CommandExecError exception if the check fails, + or returns the output of the check. + """ + # test_string = "hello" + args = [ + "/code/tcppingasym.sh", + ip, + ] + + command = ' '.join(args) + + tcp_asym_check = partial(self.execute, command) + return tcp_asym_check + + @debug_failures + def check_can_tcp(self, ip, retries=0): + """ + Execute a tcp check from this ip to the other ip. + Assert that a ip can connect to another ip. + Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently unreachable. + + :param ip: The ip to check connectivity to. + :param retries: The number of retries. + :return: None. + """ + try: + retry_until_success(self._get_tcp_function(ip), + retries=retries, + ex_class=CommandExecError) + except CommandExecError: + return False + + return True + + @debug_failures + def check_can_tcp_asym(self, ip, retries=0): + """ + Execute a tcp check from this ip to the other ip. + Assert that a ip can connect to another ip. + Use retries to allow for convergence. + Use of this method assumes the network will be transitioning from a + state where the destination is currently unreachable. + :param ip: The ip to check connectivity to. + :param retries: The number of retries. + :return: None. + """ + try: + retry_until_success(self._get_tcp_asym_function(ip), + retries=retries, + ex_class=CommandExecError) + except CommandExecError: + return False + + return True + + @debug_failures + def check_cant_tcp(self, ip, retries=0): + """ + Execute a ping from this workload to an ip. + Assert that the workload cannot connect to an IP using tcp. + Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently reachable. + + :param ip: The ip to check connectivity to. + :param retries: The number of retries. + :return: None. + """ + tcp_check = self._get_tcp_function(ip) + + def cant_tcp(): + try: + tcp_check() + except CommandExecError: + pass + else: + raise _PingError() + + try: + retry_until_success(cant_tcp, + retries=retries, + ex_class=_PingError) + except _PingError: + return False + + return True + + def _get_udp_function(self, ip): + """ + Return a function to check udp connectivity to another ip. + + :param ip: The ip to check against. + :return: A partial function that can be executed to perform the check. + The function raises a CommandExecError exception if the check fails, + or returns the output of the check. + """ + args = [ + "/code/udpping.sh", + ip, + ] + + command = ' '.join(args) + + udp_check = partial(self.execute, command) + return udp_check + + @debug_failures + def check_can_udp(self, ip, retries=0): + """ + Execute a udp check from this workload to an ip. + Assert that this workload can connect to another ip. + Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently unreachable. + + :param ip: The ip to check connectivity to. + :param retries: The number of retries. + :return: None. + """ + try: + retry_until_success(self._get_udp_function(ip), + retries=retries, + ex_class=CommandExecError) + except CommandExecError: + return False + return True + + @debug_failures + def check_cant_udp(self, ip, retries=0): + """ + Execute a udp check from this workload to the ip. Assert that + the workload cannot connect via udp to an IP. + Use retries to allow for convergence. + + Use of this method assumes the network will be transitioning from a + state where the destination is currently reachable. + + :param ip: The ip to check connectivity to. + :param retries: The number of retries. + :return: None. + """ + udp_check = self._get_udp_function(ip) + + def cant_udp(): + try: + udp_check() + except CommandExecError: + pass + else: + raise _PingError() + + try: + retry_until_success(cant_udp, + retries=retries, + ex_class=_PingError) + except _PingError: + return False + + return True + + def __str__(self): + return self.name + + +class _PingError(Exception): + pass diff --git a/tests/ut/__init__.py b/tests/ut/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/ut/test_log_parsing.py b/tests/ut/test_log_parsing.py new file mode 100644 index 000000000..267de647e --- /dev/null +++ b/tests/ut/test_log_parsing.py @@ -0,0 +1,142 @@ +# Copyright 2016 Tigera, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nose_parameterized import parameterized + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost +from tests.st.utils.utils import ETCD_CA, ETCD_CERT, \ + ETCD_KEY, ETCD_HOSTNAME_SSL, ETCD_SCHEME, get_ip + +_log = logging.getLogger(__name__) +_log.setLevel(logging.DEBUG) + +POST_DOCKER_COMMANDS = [] + +if ETCD_SCHEME == "https": + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " \ + "--cluster-store-opt kv.cacertfile=%s " \ + "--cluster-store-opt kv.certfile=%s " \ + "--cluster-store-opt kv.keyfile=%s " % \ + (ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, + ETCD_KEY) +else: + ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ + get_ip() + +felix_logfile = "/var/log/calico/felix/current" +before_data = """2017-01-12 19:19:04.419 [INFO][87] ipip_mgr.go 75: Setting local IPv4 address on link. addr=192.168.151.0 link="tunl0" +2017-01-12 19:19:04.419 [INFO][87] int_dataplane.go 389: Received interface update msg=&intdataplane.ifaceUpdate{Name:"lo", State:"up"} +2017-01-12 19:19:04.419 [INFO][87] ipip_mgr.go 95: Removing old address addr=192.168.151.0 link="tunl0" oldAddr=192.168.151.0/32 tunl0 +2017-01-12 19:19:04.416 [INFO][87] syncer.go 247: etcd watch thread started. +2017-01-12 19:19:04.419 [INFO][87] int_dataplane.go 378: Received update from calculation graph msg=config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: config: +2017-01-12 19:19:04.419 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"lo", Addrs:set.mapSet{"127.0.0.1":set.empty{}, "::1":set.empty{}}} +2017-01-12 19:19:04.419 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"ens4", Addrs:set.mapSet{"fe80::4001:aff:fef0:30":set.empty{}, "10.240.0.48":set.empty{}}} +2017-01-12 19:19:04.419 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"calic50350b9abf", Addrs:set.mapSet{"fe80::a077:a7ff:fe1c:8436":set.empty{}}} +2017-01-12 19:19:04.420 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"calie9054722202", Addrs:set.mapSet{"fe80::d046:64ff:fe86:c21":set.empty{}}} +2017-01-12 19:19:04.420 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"cali076a4d2f51a", Addrs:set.mapSet{"fe80::2c6e:f7ff:fe0d:2b86":set.empty{}}} +2017-01-12 19:19:04.420 [INFO][87] syncer.go 261: Polled etcd for initial watch index. index=0x3f85 +2017-01-12 19:19:04.420 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"cali76b1299437f", Addrs:set.mapSet{"fe80::30f3:a9ff:fe6e:2d22":set.empty{}}} +2017-01-12 19:19:04.420 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"cali477d4934e36", Addrs:set.mapSet{"fe80::70d6:51ff:feca:1b41":set.empty{}}} +2017-01-12 19:19:04.419 [INFO][87] iface_monitor.go 120: Netlink address update. addr="192.168.151.0" exists=false ifIndex=14 +2017-01-12 19:19:04.421 [INFO][87] int_dataplane.go 389: Received interface update msg=&intdataplane.ifaceUpdate{Name:"ens4", State:"up"} +2017-01-12 19:19:04.421 [INFO][87] int_dataplane.go 288: Linux interface addrs changed. addrs=set.mapSet{} ifaceName="tunl0" +2017-01-12 19:19:04.420 [INFO][87] syncer.go 461: Watcher out-of-sync, starting to track deletions +2017-01-12 19:19:04.421 [INFO][87] int_dataplane.go 389: Received interface update msg=&intdataplane.ifaceUpdate{Name:"calic50350b9abf", State:"up"} +2017-01-12 19:19:04.421 [INFO][87] ipip_mgr.go 103: Address wasn't present, adding it. addr=192.168.151.0 link="tunl0" +2017-01-12 19:19:04.421 [INFO][87] int_dataplane.go 398: Received interface addresses update msg=&intdataplane.ifaceAddrsUpdate{Name:"cali3b05e50a7e8", Addrs:set.mapSet{"fe80::94d3:2bff:fe26:e4fe":set.empty{}}} +2017-01-12 19:19:04.421 [INFO][87] iface_monitor.go 120: Netlink address update. addr="192.168.151.0" exists=true ifIndex=14 +2017-01-12 19:19:04.421 [INFO][87] syncer.go 500: Watcher is out-of-sync but no snapshot in progress, starting one. +""" + + +class LogParsing(TestBase): + @classmethod + def setUpClass(cls): + cls.log_banner("TEST SET UP STARTING: %s", cls.__name__) + + cls.hosts = [] + cls.hosts.append(DockerHost("host1", + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False)) + cls.hosts.append(DockerHost("host2", + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=POST_DOCKER_COMMANDS, + start_calico=False)) + for host in cls.hosts: + host.execute("mkdir -p /var/log/calico/felix/") + host.writefile(felix_logfile, before_data) + host.attach_log_analyzer() + cls.expect_errors = False + + @classmethod + def tearDownClass(cls): + # Tidy up + for host in cls.hosts: + host.cleanup() + del host + + def setUp(self): + self.log_banner("starting %s", self._testMethodName) + + _log.debug("Reset Log Analyzers") + for host in self.hosts: + host.log_analyzer.reset() + + def tearDown(self): + _log.info("Checking logs for exceptions") + failed = False + if self.expect_errors: + try: + for host in self.hosts: + host.log_analyzer.check_logs_for_exceptions() + _log.debug("No exceptions found, setting failed=True") + failed = True + except AssertionError: + _log.debug("Hit the error we expected. Good.") + _log.debug("Failed = %s", failed) + assert not failed, "Did not hit error! Fail." + else: + for host in self.hosts: + host.log_analyzer.check_logs_for_exceptions() + + def test_no_logs(self): + """ + Tests that the scenario with no new logs works OK + """ + _log.debug("\n") + self.expect_errors = False + + @parameterized.expand([ + ("ERROR", + "2017-01-12 19:19:05.421 [ERROR][87] syncer.go 500: Watcher is out-of-sync.", + True), + ("INFO", + "2017-01-12 19:19:05.421 [INFO][87] syncer.go 500: Watcher is out-of-sync.", + False), + ]) + def test_newlog(self, name, log, expect_error): + """ + Tests that the scenario with a new log works OK + """ + self.__name__ = name + _log.debug("\n") + self.expect_errors = expect_error + self.hosts[0].execute("echo %s >> %s" % (log, felix_logfile)) + + +class IpNotFound(Exception): + pass diff --git a/workload/Dockerfile b/workload/Dockerfile new file mode 100644 index 000000000..4145451aa --- /dev/null +++ b/workload/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.4 +RUN apk add --no-cache \ + python \ + netcat-openbsd +COPY udpping.sh tcpping.sh responder.py /code/ +WORKDIR /code/ +RUN chmod +x udpping.sh && chmod +x tcpping.sh +CMD ["python", "responder.py"] diff --git a/workload/responder.py b/workload/responder.py new file mode 100644 index 000000000..a5c6b6749 --- /dev/null +++ b/workload/responder.py @@ -0,0 +1,106 @@ +import logging +import SocketServer +import time + +logging.basicConfig(level=logging.DEBUG, + format='%(name)s: %(message)s', + ) + +logger = logging.getLogger(__name__) + + +class EchoRequestHandlerTCP(SocketServer.BaseRequestHandler): + def handle(self): + logger.debug('handle') + # Echo the back to the client + data = self.request.recv(1024) + logger.debug('received (tcp) from %s: "%s"', + self.client_address, data) + self.request.send(data) + return + + +class EchoRequestHandlerUDP(SocketServer.BaseRequestHandler): + def handle(self): + logger.debug('handle') + + # Echo the back to the client + data = self.request[0] + socket = self.request[1] + logger.debug('received (udp) from %s: "%s"', + self.client_address, data) + socket.sendto(data, self.client_address) + return + + +class EchoServerTCP(SocketServer.TCPServer): + def serve_forever(self): + logger.info('waiting for tcp request') + while True: + self.handle_request() + return + + +class EchoServerUDP(SocketServer.UDPServer): + def serve_forever(self): + logger.info('waiting for udp request') + while True: + self.handle_request() + return + + +if __name__ == '__main__': + import socket + import threading + + def check_socket(sock): + # Send the data + message = 'Hello world' + logger.debug('sending data: "%s"', message) + len_sent = sock.send(message) + + # Receive a response + logger.debug('waiting for response') + response = sock.recv(len_sent) + logger.debug('response from server: "%s"', response) + + tcp_addr = "0.0.0.0" + tcp_port = 80 + udp_addr = "0.0.0.0" + udp_port = 69 + + tcp_server = EchoServerTCP((tcp_addr, tcp_port), EchoRequestHandlerTCP) + udp_server = EchoServerUDP((udp_addr, udp_port), EchoRequestHandlerUDP) + + try: + t1 = threading.Thread(target=tcp_server.serve_forever) + t1.setDaemon(True) # don't hang on exit + t1.start() + t2 = threading.Thread(target=udp_server.serve_forever) + t2.setDaemon(True) # don't hang on exit + t2.start() + + logger.info('TCP Server on %s:%s', tcp_addr, tcp_port) + logger.info('UDP Server on %s:%s', udp_addr, udp_port) + + logger.debug('checking tcp server') + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + logger.debug('connecting to server') + s.connect((tcp_addr, tcp_port)) + check_socket(s) + s.close() + + logger.debug('checking udp server') + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + logger.debug('connecting to server') + s.connect((udp_addr, udp_port)) + check_socket(s) + s.close() + while True: + time.sleep(10) + finally: + # Clean up + logger.debug('done') + tcp_server.socket.close() + udp_server.socket.close() + logger.debug('closed sockets') diff --git a/workload/tcpping.sh b/workload/tcpping.sh new file mode 100644 index 000000000..63425be97 --- /dev/null +++ b/workload/tcpping.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo hello | nc -w1 $1 80 | grep hello diff --git a/workload/udpping.sh b/workload/udpping.sh new file mode 100644 index 000000000..bd784b33f --- /dev/null +++ b/workload/udpping.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo hello | nc -u -w1 $1 69 | grep hello