From 55fa65aca8ba53e186050cb614e6b463127bc930 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 08:50:47 +0200 Subject: [PATCH 01/14] added import scripts --- .circleci/config.yml | 10 +- .gitignore | 1 + charts/polkadot/templates/alertrules.yaml | 72 ------------ scripts/alertrules/import-alertrule-test.sh | 6 + scripts/alertrules/import-alertrule.sh | 12 ++ scripts/alertrules/import-substrate.sh | 6 + scripts/alertrules/import.sh | 9 ++ scripts/alertrules/template/postfix.yaml | 1 + scripts/alertrules/template/prefix.yaml | 7 ++ scripts/alertrules/vars.sh | 11 ++ tests/prometheus/polkadot/alertrules.yaml | 123 -------------------- 11 files changed, 60 insertions(+), 198 deletions(-) create mode 100644 .gitignore create mode 100755 scripts/alertrules/import-alertrule-test.sh create mode 100755 scripts/alertrules/import-alertrule.sh create mode 100755 scripts/alertrules/import-substrate.sh create mode 100755 scripts/alertrules/import.sh create mode 100644 scripts/alertrules/template/postfix.yaml create mode 100644 scripts/alertrules/template/prefix.yaml create mode 100755 scripts/alertrules/vars.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a9790f..5e8083d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: command: | /scripts/publish-chart.sh - testPrometheusRules: + importAndTestPrometheusRules: docker: - image: web3f/ci-commons:v2.1.3 steps: @@ -36,6 +36,10 @@ jobs: wget -O /tmp/prometheus.tgz https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz tar -xvf /tmp/prometheus.tgz prometheus-${PROM_VER}.linux-amd64/promtool -C /tmp mv /tmp/prometheus-$PROM_VER.linux-amd64/promtool /usr/local/bin/ + - run: + name: Import external alertrules + command: | + scripts/alertrules/import.sh - run: command: | scripts/test_prometheus_rules.sh @@ -49,14 +53,14 @@ workflows: filters: tags: only: /.*/ - - testPrometheusRules: + - importAndTestPrometheusRules: filters: tags: only: /.*/ - publishChart: requires: - buildImage - - testPrometheusRules + - importAndTestPrometheusRules filters: branches: ignore: /.*/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..686f157 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +alertrules-substrate.yaml \ No newline at end of file diff --git a/charts/polkadot/templates/alertrules.yaml b/charts/polkadot/templates/alertrules.yaml index 8d1d670..f8d62c7 100644 --- a/charts/polkadot/templates/alertrules.yaml +++ b/charts/polkadot/templates/alertrules.yaml @@ -9,76 +9,4 @@ spec: groups: - name: polkadot.rules rules: - - alert: PolkadotLowNumberOfPeersShort - annotations: - message: 'The node has less than 3 peers for 3 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: polkadot_sub_libp2p_peers_count < 3 - for: 3m - labels: - severity: warning - origin: {{ .Values.deploymentName }} - - alert: PolkadotLowNumberOfPeersLong - annotations: - message: 'The node has less than 3 peers for 15 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: polkadot_sub_libp2p_peers_count < 3 - for: 15m - labels: - severity: critical - origin: {{ .Values.deploymentName }} - - alert: PolkadotTransactionQueueSizeShort - annotations: - message: 'The node has more than 10 transactions in the queue for more than 10 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: polkadot_sub_txpool_validations_scheduled - polkadot_sub_txpool_validations_finished > 10 - for: 10m - labels: - severity: warning - origin: {{ .Values.deploymentName }} - - alert: PolkadotTransactionQueueSizeLong - annotations: - message: 'The node has more than 10 transactions in the queue for more than 30 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: polkadot_sub_txpool_validations_scheduled - polkadot_sub_txpool_validations_finished > 10 - for: 30m - labels: - severity: critical - origin: {{ .Values.deploymentName }} - - alert: PolkadotLowNumberOfNewBlocksShort - annotations: - message: 'The number of new blocks has not increased for the last 3 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: increase(polkadot_block_height{status="best"}[1m]) == 0 - for: 3m - labels: - severity: warning - origin: {{ .Values.deploymentName }} - - alert: PolkadotLowNumberOfNewBlocksLong - annotations: - message: 'The number of new blocks has not increased for the last 10 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: increase(polkadot_block_height{status="best"}[1m]) == 0 - for: 10m - labels: - severity: critical - origin: {{ .Values.deploymentName }} - - alert: PolkadotLowNumberOfFinalizedBlocksShort - annotations: - message: 'The number of finalized blocks has not increased for the last 3 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: increase(polkadot_block_height{status="finalized"}[1m]) == 0 - for: 3m - labels: - severity: warning - origin: {{ .Values.deploymentName }} - - alert: PolkadotLowNumberOfFinalizedBlocksLong - annotations: - message: 'The number of finalized blocks has not increased for the last 10 minutes' - # runbook_url: "https://github.com/w3f/infrastructure/wiki/" - expr: increase(polkadot_block_height{status="finalized"}[1m]) == 0 - for: 10m - labels: - severity: critical - origin: {{ .Values.deploymentName }} {{ end }} diff --git a/scripts/alertrules/import-alertrule-test.sh b/scripts/alertrules/import-alertrule-test.sh new file mode 100755 index 0000000..df20130 --- /dev/null +++ b/scripts/alertrules/import-alertrule-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +CURRENT_DIR=$(dirname "$0") +source ${CURRENT_DIR}/vars.sh + +yq w ${SUBSTRATE_RULES_FOLDER}/alerting-rule-tests.yaml "tests[*].alert_rule_test[*].exp_alerts[*].exp_labels.origin" testnet-0 | sed 's/{}//g' - > ${PROJECT_ROOT}/${TEST_FOLDER}/${TEST_FILE_NAME} \ No newline at end of file diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh new file mode 100755 index 0000000..8a8689a --- /dev/null +++ b/scripts/alertrules/import-alertrule.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +CURRENT_DIR=$(dirname "$0") +source ${CURRENT_DIR}/vars.sh + +cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ $labels.instance }}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value }}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} +mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} + +echo "imported new ${RULE_FILE_NAME}" +cat ${PROJECT_ROOT}/${CHART_FOLDER}/${RULE_FILE_NAME} \ No newline at end of file diff --git a/scripts/alertrules/import-substrate.sh b/scripts/alertrules/import-substrate.sh new file mode 100755 index 0000000..2fab1ad --- /dev/null +++ b/scripts/alertrules/import-substrate.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +CURRENT_DIR=$(dirname "$0") +source ${CURRENT_DIR}/vars.sh + +git colone -b ${SUBSTRATE_VER} https://github.com/paritytech/substrate.git ${SUBSTRATE_FOLDER} diff --git a/scripts/alertrules/import.sh b/scripts/alertrules/import.sh new file mode 100755 index 0000000..da04dc6 --- /dev/null +++ b/scripts/alertrules/import.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +CURRENT_DIR=$(dirname "$0") + +${CURRENT_DIR}/import-substrate.sh +${CURRENT_DIR}/import-alertrule.sh +${CURRENT_DIR}/import-alertrule-test.sh + +rm -rf ${SUBSTRATE_FOLDER}/substrate \ No newline at end of file diff --git a/scripts/alertrules/template/postfix.yaml b/scripts/alertrules/template/postfix.yaml new file mode 100644 index 0000000..a757b67 --- /dev/null +++ b/scripts/alertrules/template/postfix.yaml @@ -0,0 +1 @@ +{{ end }} \ No newline at end of file diff --git a/scripts/alertrules/template/prefix.yaml b/scripts/alertrules/template/prefix.yaml new file mode 100644 index 0000000..8c5cd95 --- /dev/null +++ b/scripts/alertrules/template/prefix.yaml @@ -0,0 +1,7 @@ +{{ if .Values.monitoring }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ .Release.Name }}-alertrules-substrate + labels: +{{ toYaml .Values.alertRulesLabels | indent 4 }} diff --git a/scripts/alertrules/vars.sh b/scripts/alertrules/vars.sh new file mode 100755 index 0000000..35f265b --- /dev/null +++ b/scripts/alertrules/vars.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +SUBSTRATE_VER=v2.0.0-rc6 +PROJECT_ROOT=`git rev-parse --show-toplevel` +TEST_FOLDER="tests/prometheus/polkadot" +CHART_FOLDER="charts/polkadot/templates" +RULE_FILE_NAME="alertrules-substrate.yaml" +TEST_FILE_NAME="alertrules-substrate.yaml" +SUBSTRATE_FOLDER="/tmp" +#SUBSTRATE_FOLDER=${PROJECT_ROOT} +SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/substrate/.maintain/monitoring/alerting-rules \ No newline at end of file diff --git a/tests/prometheus/polkadot/alertrules.yaml b/tests/prometheus/polkadot/alertrules.yaml index 9e1ef62..d74e4c3 100644 --- a/tests/prometheus/polkadot/alertrules.yaml +++ b/tests/prometheus/polkadot/alertrules.yaml @@ -20,126 +20,3 @@ tests: values: '0+20x5 100+0x5' # 0 20 40 60 80 100 100 100 100 100 100 alert_rule_test: - # Test LowNumberOfPeersShort alert - - eval_time: 3m # Values: 3 2 2 - alertname: PolkadotLowNumberOfPeersShort - exp_alerts: - - eval_time: 4m # Values: 2 2 2 - alertname: PolkadotLowNumberOfPeersShort - exp_alerts: - - exp_labels: - severity: warning - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - exp_annotations: - message: "The node has less than 3 peers for 3 minutes" - - # Test LowNumberOfPeersLong alert - - eval_time: 15m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 - alertname: PolkadotLowNumberOfPeersLong - exp_alerts: - - eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 - alertname: PolkadotLowNumberOfPeersLong - exp_alerts: - - exp_labels: - severity: critical - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - exp_annotations: - message: "The node has less than 3 peers for 15 minutes" - - # Test TransactionQueueSizeShort alert - - eval_time: 10m - alertname: PolkadotTransactionQueueSizeShort - exp_alerts: - - eval_time: 11m - alertname: PolkadotTransactionQueueSizeShort - exp_alerts: - - exp_labels: - severity: warning - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - exp_annotations: - message: "The node has more than 10 transactions in the queue for more than 10 minutes" - - # Test TransactionQueueSizeLong alert - - eval_time: 30m - alertname: PolkadotTransactionQueueSizeLong - exp_alerts: - - eval_time: 31m - alertname: PolkadotTransactionQueueSizeLong - exp_alerts: - - exp_labels: - severity: critical - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - exp_annotations: - message: "The node has more than 10 transactions in the queue for more than 30 minutes" - - # Test LowNumberOfNewBlocksShort alert - - eval_time: 6m - alertname: PolkadotLowNumberOfNewBlocksShort - exp_alerts: - - eval_time: 7m - alertname: PolkadotLowNumberOfNewBlocksShort - exp_alerts: - - exp_labels: - severity: warning - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - status: best - exp_annotations: - message: "The number of new blocks has not increased for the last 3 minutes" - - # Test LowNumberOfNewBlocksLong alert - - eval_time: 13m - alertname: PolkadotLowNumberOfNewBlocksLong - exp_alerts: - - eval_time: 14m - alertname: PolkadotLowNumberOfNewBlocksLong - exp_alerts: - - exp_labels: - severity: critical - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - status: best - exp_annotations: - message: "The number of new blocks has not increased for the last 10 minutes" - - # Test LowNumberOfFinalizedBlocksShort alert - - eval_time: 6m - alertname: PolkadotLowNumberOfFinalizedBlocksShort - exp_alerts: - - eval_time: 7m - alertname: PolkadotLowNumberOfFinalizedBlocksShort - exp_alerts: - - exp_labels: - severity: warning - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - status: finalized - exp_annotations: - message: "The number of finalized blocks has not increased for the last 3 minutes" - - # Test LowNumberOfFinalizedBlocksLong alert - - eval_time: 13m - alertname: PolkadotLowNumberOfFinalizedBlocksLong - exp_alerts: - - eval_time: 14m - alertname: PolkadotLowNumberOfFinalizedBlocksLong - exp_alerts: - - exp_labels: - severity: critical - origin: testnet-0 - pod: polkadot-abcdef01234-abcdef - job: polkadot - status: finalized - exp_annotations: - message: "The number of finalized blocks has not increased for the last 10 minutes" From 94539e78bd0005dffa19ac307c05914b5607187c Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 08:55:12 +0200 Subject: [PATCH 02/14] fixed typo --- scripts/alertrules/import-substrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-substrate.sh b/scripts/alertrules/import-substrate.sh index 2fab1ad..7a7bd91 100755 --- a/scripts/alertrules/import-substrate.sh +++ b/scripts/alertrules/import-substrate.sh @@ -3,4 +3,4 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh -git colone -b ${SUBSTRATE_VER} https://github.com/paritytech/substrate.git ${SUBSTRATE_FOLDER} +git clone -b ${SUBSTRATE_VER} https://github.com/paritytech/substrate.git ${SUBSTRATE_FOLDER} From d31383675ebdf1edeb5df94a00e234b780d494c4 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 09:02:13 +0200 Subject: [PATCH 03/14] fixed folder path --- scripts/alertrules/vars.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/alertrules/vars.sh b/scripts/alertrules/vars.sh index 35f265b..3fd5e71 100755 --- a/scripts/alertrules/vars.sh +++ b/scripts/alertrules/vars.sh @@ -6,6 +6,5 @@ TEST_FOLDER="tests/prometheus/polkadot" CHART_FOLDER="charts/polkadot/templates" RULE_FILE_NAME="alertrules-substrate.yaml" TEST_FILE_NAME="alertrules-substrate.yaml" -SUBSTRATE_FOLDER="/tmp" -#SUBSTRATE_FOLDER=${PROJECT_ROOT} -SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/substrate/.maintain/monitoring/alerting-rules \ No newline at end of file +SUBSTRATE_FOLDER="/tmp/substrate" +SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/.maintain/monitoring/alerting-rules \ No newline at end of file From de1f636d2e8ccd9b9a3e06dc976e9b5f8c0516ec Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 09:23:35 +0200 Subject: [PATCH 04/14] improved regex --- scripts/alertrules/import-alertrule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index 8a8689a..af62575 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ $labels.instance }}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value }}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From 433d5e376997f964e600c6e69edabb94f8216726 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 09:28:24 +0200 Subject: [PATCH 05/14] improved regex --- scripts/alertrules/import-alertrule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index af62575..6cafe5b 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From 4617cf31f5a11db20c4483025e79854e72fa73e1 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 09:55:37 +0200 Subject: [PATCH 06/14] try to check cache issue --- scripts/alertrules/vars.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/alertrules/vars.sh b/scripts/alertrules/vars.sh index 3fd5e71..af68b50 100755 --- a/scripts/alertrules/vars.sh +++ b/scripts/alertrules/vars.sh @@ -4,7 +4,7 @@ SUBSTRATE_VER=v2.0.0-rc6 PROJECT_ROOT=`git rev-parse --show-toplevel` TEST_FOLDER="tests/prometheus/polkadot" CHART_FOLDER="charts/polkadot/templates" -RULE_FILE_NAME="alertrules-substrate.yaml" -TEST_FILE_NAME="alertrules-substrate.yaml" +RULE_FILE_NAME="alertrules-substrate2.yaml" +TEST_FILE_NAME="alertrules-substrate2.yaml" SUBSTRATE_FOLDER="/tmp/substrate" SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/.maintain/monitoring/alerting-rules \ No newline at end of file From bbff116333dfd862d6a0cd58643244a6a85b9737 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 10:47:29 +0200 Subject: [PATCH 07/14] testing problem --- scripts/alertrules/import-alertrule.sh | 2 +- scripts/alertrules/vars.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index 6cafe5b..f30fb6f 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/Less than one new block per minute on instance/test/g' - | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} diff --git a/scripts/alertrules/vars.sh b/scripts/alertrules/vars.sh index af68b50..3fd5e71 100755 --- a/scripts/alertrules/vars.sh +++ b/scripts/alertrules/vars.sh @@ -4,7 +4,7 @@ SUBSTRATE_VER=v2.0.0-rc6 PROJECT_ROOT=`git rev-parse --show-toplevel` TEST_FOLDER="tests/prometheus/polkadot" CHART_FOLDER="charts/polkadot/templates" -RULE_FILE_NAME="alertrules-substrate2.yaml" -TEST_FILE_NAME="alertrules-substrate2.yaml" +RULE_FILE_NAME="alertrules-substrate.yaml" +TEST_FILE_NAME="alertrules-substrate.yaml" SUBSTRATE_FOLDER="/tmp/substrate" SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/.maintain/monitoring/alerting-rules \ No newline at end of file From 90b795d2decaf1d196a373b21bb6bca309bc350e Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 10:53:54 +0200 Subject: [PATCH 08/14] test --- scripts/alertrules/import-alertrule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index f30fb6f..f6dc596 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/Less than one new block per minute on instance/test/g' - | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/Less than one new block per minute on instance/test/gm' - | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/gm' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/gm' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/gm" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From e7a8e1882b9b4a55550fd629e4faec98a504951d Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 11:03:23 +0200 Subject: [PATCH 09/14] test fix --- .circleci/config.yml | 2 +- scripts/alertrules/import-alertrule.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e8083d..d7d26b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: - run: name: Install missing dependencies command: | - YQ_VER=3.3.0 + YQ_VER=3.3.2 wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VER}/yq_linux_amd64 chmod +x /usr/local/bin/yq diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index f6dc596..b92fa97 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/Less than one new block per minute on instance/test/gm' - | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/gm' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/gm' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/gm" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/gm' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/gm' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/gm" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From 94e1d950579d9e198ef8f9e9745ae888835c9eae Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 11:05:07 +0200 Subject: [PATCH 10/14] test fix --- scripts/alertrules/import-alertrule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index b92fa97..6cafe5b 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/gm' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/gm' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/gm" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From 29da4d2915b154fef039dc66be07a8feaec98f19 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 11:09:18 +0200 Subject: [PATCH 11/14] fixed updating the yq version, trying to simplify the regex --- scripts/alertrules/import-alertrule.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh index 6cafe5b..8a8689a 100755 --- a/scripts/alertrules/import-alertrule.sh +++ b/scripts/alertrules/import-alertrule.sh @@ -4,7 +4,7 @@ CURRENT_DIR=$(dirname "$0") source ${CURRENT_DIR}/vars.sh cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ \$labels\.instance.*\n*\s*\t*}}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value.*\n*\s*\t*}}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} +yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ $labels.instance }}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value }}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} From 47e103393bcb7e34e4d7da31cbe2e919d784e9df Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 11:14:10 +0200 Subject: [PATCH 12/14] added import run to publish chart --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7d26b5..2479f40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,10 @@ jobs: - image: web3f/ci-commons:v2.1.3 steps: - checkout + - run: + name: Import external alertrules + command: | + scripts/alertrules/import.sh - run: command: | /scripts/publish-chart.sh From 4ce52c4dd1cfdee4115428f965fd57b5337cf059 Mon Sep 17 00:00:00 2001 From: ironoa Date: Tue, 1 Sep 2020 11:21:22 +0200 Subject: [PATCH 13/14] updated chart version --- charts/polkadot-base-services/Chart.yaml | 2 +- charts/polkadot-secrets/Chart.yaml | 2 +- charts/polkadot/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/polkadot-base-services/Chart.yaml b/charts/polkadot-base-services/Chart.yaml index b3b85a5..f2e990e 100644 --- a/charts/polkadot-base-services/Chart.yaml +++ b/charts/polkadot-base-services/Chart.yaml @@ -1,4 +1,4 @@ description: Base services for Polkadot Node chart. name: polkadot-base-services -version: v0.30.1 +version: v0.32.0 apiVersion: v2 diff --git a/charts/polkadot-secrets/Chart.yaml b/charts/polkadot-secrets/Chart.yaml index 7fdc650..8a8be39 100644 --- a/charts/polkadot-secrets/Chart.yaml +++ b/charts/polkadot-secrets/Chart.yaml @@ -1,4 +1,4 @@ description: Polkadot secrets name: polkadot-secrets -version: v0.30.1 +version: v0.32.0 apiVersion: v2 diff --git a/charts/polkadot/Chart.yaml b/charts/polkadot/Chart.yaml index 6da429e..8f7b6f4 100644 --- a/charts/polkadot/Chart.yaml +++ b/charts/polkadot/Chart.yaml @@ -1,4 +1,4 @@ description: Polkadot Node chart. name: polkadot -version: v0.30.1 +version: v0.32.0 apiVersion: v2 From fb972deb395b643ca7afdc9bd1be7c868644f30f Mon Sep 17 00:00:00 2001 From: ironoa Date: Wed, 2 Sep 2020 08:23:04 +0200 Subject: [PATCH 14/14] migrated substrate-alertrules chart to dedicated repo --- .circleci/config.yml | 34 +++++---------------- .gitignore | 1 - scripts/alertrules/import-alertrule-test.sh | 6 ---- scripts/alertrules/import-alertrule.sh | 12 -------- scripts/alertrules/import-substrate.sh | 6 ---- scripts/alertrules/import.sh | 9 ------ scripts/alertrules/template/postfix.yaml | 1 - scripts/alertrules/template/prefix.yaml | 7 ----- scripts/alertrules/vars.sh | 10 ------ 9 files changed, 8 insertions(+), 78 deletions(-) delete mode 100755 scripts/alertrules/import-alertrule-test.sh delete mode 100755 scripts/alertrules/import-alertrule.sh delete mode 100755 scripts/alertrules/import-substrate.sh delete mode 100755 scripts/alertrules/import.sh delete mode 100644 scripts/alertrules/template/postfix.yaml delete mode 100644 scripts/alertrules/template/prefix.yaml delete mode 100755 scripts/alertrules/vars.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 2479f40..1b74b32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 jobs: buildImage: docker: - - image: web3f/ci-commons:v2.1.3 + - image: web3f/ci-commons:v2.4.0 steps: - checkout - setup_remote_docker @@ -13,37 +13,18 @@ jobs: publishChart: docker: - - image: web3f/ci-commons:v2.1.3 + - image: web3f/ci-commons:v2.4.0 steps: - checkout - - run: - name: Import external alertrules - command: | - scripts/alertrules/import.sh - run: command: | /scripts/publish-chart.sh - importAndTestPrometheusRules: + testPrometheusRules: docker: - - image: web3f/ci-commons:v2.1.3 + - image: web3f/ci-commons:v2.4.0 steps: - checkout - - run: - name: Install missing dependencies - command: | - YQ_VER=3.3.2 - wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VER}/yq_linux_amd64 - chmod +x /usr/local/bin/yq - - PROM_VER=2.18.1 - wget -O /tmp/prometheus.tgz https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz - tar -xvf /tmp/prometheus.tgz prometheus-${PROM_VER}.linux-amd64/promtool -C /tmp - mv /tmp/prometheus-$PROM_VER.linux-amd64/promtool /usr/local/bin/ - - run: - name: Import external alertrules - command: | - scripts/alertrules/import.sh - run: command: | scripts/test_prometheus_rules.sh @@ -57,16 +38,17 @@ workflows: filters: tags: only: /.*/ - - importAndTestPrometheusRules: + - testPrometheusRules: filters: tags: only: /.*/ - publishChart: + context: github-bot requires: - buildImage - - importAndTestPrometheusRules + - testPrometheusRules filters: branches: ignore: /.*/ tags: - only: /v[0-9]+(\.[0-9]+)*/ + only: /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/ diff --git a/.gitignore b/.gitignore index 686f157..e69de29 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +0,0 @@ -alertrules-substrate.yaml \ No newline at end of file diff --git a/scripts/alertrules/import-alertrule-test.sh b/scripts/alertrules/import-alertrule-test.sh deleted file mode 100755 index df20130..0000000 --- a/scripts/alertrules/import-alertrule-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -CURRENT_DIR=$(dirname "$0") -source ${CURRENT_DIR}/vars.sh - -yq w ${SUBSTRATE_RULES_FOLDER}/alerting-rule-tests.yaml "tests[*].alert_rule_test[*].exp_alerts[*].exp_labels.origin" testnet-0 | sed 's/{}//g' - > ${PROJECT_ROOT}/${TEST_FOLDER}/${TEST_FILE_NAME} \ No newline at end of file diff --git a/scripts/alertrules/import-alertrule.sh b/scripts/alertrules/import-alertrule.sh deleted file mode 100755 index 8a8689a..0000000 --- a/scripts/alertrules/import-alertrule.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CURRENT_DIR=$(dirname "$0") -source ${CURRENT_DIR}/vars.sh - -cat ${CURRENT_DIR}/template/prefix.yaml > ${RULE_FILE_NAME} -yq p ${SUBSTRATE_RULES_FOLDER}/alerting-rules.yaml spec | sed 's/{{ $labels.instance }}/{{`{{ $labels.instance }}`}}/g' - | sed 's/{{ $value }}/{{`{{ $value }}`}}/g' - | yq w - "spec.groups[*].rules[*].labels.origin" "{{ .Values.deploymentName }}" | sed "s/'{{ .Values.deploymentName }}'/{{ .Values.deploymentName }}/g" - >> ${RULE_FILE_NAME} -cat ${CURRENT_DIR}/template/postfix.yaml >> ${RULE_FILE_NAME} -mv ${RULE_FILE_NAME} ${PROJECT_ROOT}/${CHART_FOLDER} - -echo "imported new ${RULE_FILE_NAME}" -cat ${PROJECT_ROOT}/${CHART_FOLDER}/${RULE_FILE_NAME} \ No newline at end of file diff --git a/scripts/alertrules/import-substrate.sh b/scripts/alertrules/import-substrate.sh deleted file mode 100755 index 7a7bd91..0000000 --- a/scripts/alertrules/import-substrate.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -CURRENT_DIR=$(dirname "$0") -source ${CURRENT_DIR}/vars.sh - -git clone -b ${SUBSTRATE_VER} https://github.com/paritytech/substrate.git ${SUBSTRATE_FOLDER} diff --git a/scripts/alertrules/import.sh b/scripts/alertrules/import.sh deleted file mode 100755 index da04dc6..0000000 --- a/scripts/alertrules/import.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -CURRENT_DIR=$(dirname "$0") - -${CURRENT_DIR}/import-substrate.sh -${CURRENT_DIR}/import-alertrule.sh -${CURRENT_DIR}/import-alertrule-test.sh - -rm -rf ${SUBSTRATE_FOLDER}/substrate \ No newline at end of file diff --git a/scripts/alertrules/template/postfix.yaml b/scripts/alertrules/template/postfix.yaml deleted file mode 100644 index a757b67..0000000 --- a/scripts/alertrules/template/postfix.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ end }} \ No newline at end of file diff --git a/scripts/alertrules/template/prefix.yaml b/scripts/alertrules/template/prefix.yaml deleted file mode 100644 index 8c5cd95..0000000 --- a/scripts/alertrules/template/prefix.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{ if .Values.monitoring }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ .Release.Name }}-alertrules-substrate - labels: -{{ toYaml .Values.alertRulesLabels | indent 4 }} diff --git a/scripts/alertrules/vars.sh b/scripts/alertrules/vars.sh deleted file mode 100755 index 3fd5e71..0000000 --- a/scripts/alertrules/vars.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -SUBSTRATE_VER=v2.0.0-rc6 -PROJECT_ROOT=`git rev-parse --show-toplevel` -TEST_FOLDER="tests/prometheus/polkadot" -CHART_FOLDER="charts/polkadot/templates" -RULE_FILE_NAME="alertrules-substrate.yaml" -TEST_FILE_NAME="alertrules-substrate.yaml" -SUBSTRATE_FOLDER="/tmp/substrate" -SUBSTRATE_RULES_FOLDER=${SUBSTRATE_FOLDER}/.maintain/monitoring/alerting-rules \ No newline at end of file