From e41c365c1878d66731aaf2bf8ae94b696343c12c Mon Sep 17 00:00:00 2001 From: Fastly2Git Date: Mon, 13 May 2019 14:48:50 -0600 Subject: [PATCH 1/4] added --- build | 8 ++++++++ ww.metadata | 4 ++++ 2 files changed, 12 insertions(+) create mode 100755 build create mode 100644 ww.metadata diff --git a/build b/build new file mode 100755 index 0000000..c6427c5 --- /dev/null +++ b/build @@ -0,0 +1,8 @@ +#!/bin/bash -x +set -e + +BRANCH=$(git branch | grep \* | cut -d ' ' -f2) +GIT_SHA=$(git rev-parse --short HEAD) +VERSION="0.1.0-${BRANCH}-${GIT_SHA}" +docker build -t quay.io/weightwatchers/fastly-to-insights:${VERSION} . +docker push quay.io/weightwatchers/fastly-to-insights:${VERSION} diff --git a/ww.metadata b/ww.metadata new file mode 100644 index 0000000..8c4b84b --- /dev/null +++ b/ww.metadata @@ -0,0 +1,4 @@ +#Metadata Labels +sec_repository=fastly-to-insights +sec_owner=parc +sec_critical_system=true From 93cedd957eeec70eed3963046bbae768290bb8b9 Mon Sep 17 00:00:00 2001 From: Fastly2Git Date: Mon, 13 May 2019 14:49:07 -0600 Subject: [PATCH 2/4] named services and more debugging --- app.js | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index 3cd2ed2..4f4568d 100644 --- a/app.js +++ b/app.js @@ -9,17 +9,25 @@ const FASTLY_KEY = process.env.FASTLY_KEY const LIST_OF_SERVICES = process.env.SERVICES.split(" ") const request = require('request') +const call_frequency = 60 -let timestamp = 0 +let timestamp = Math.round(Date.now()/1000) - call_frequency +let send_count = 0 + +get_send_count = ( () => + send_count = send_count + 1 +) setInterval( () => { //This will create 1920 events per day, or 13440 events per week. - LIST_OF_SERVICES.map( (service) => { - pollFromFastly(service) + LIST_OF_SERVICES.map( (service_entry) => { + pollFromFastly(service_entry) }) -}, 180000) // 3 minutes +}, 1000 * call_frequency) -pollFromFastly = (service) => { +pollFromFastly = (service_entry) => { + let [service_name, service] = service_entry.split(":") + service = service || service_name; // Make it backward compatible let fastly_url = "https://rt.fastly.com/v1/channel/" + service + "/ts/" + timestamp request({ method: "GET", @@ -32,19 +40,22 @@ pollFromFastly = (service) => { if (response) { let valuableData = JSON.parse(response.body) timestamp = valuableData.Timestamp + console.log("New timestamp for " + service_name + " is " + timestamp) + console.log("SIZE=" + valuableData.Data.length + " for " + fastly_url ) valuableData.Data.map( (data) => { - batchAndSend(data.aggregated, service) + if(data.aggregated) batchAndSend(data.aggregated, service, service_name) }) } if (err) console.log(err) }) } -let batchAndSend = (aggregate, service) => { +let batchAndSend = (aggregate, service, service_name) => { let message = { - "eventType": "LogAggregate", + "eventType": "LogAggregateFastly", "service": service, + "service_name": service_name, /* These are all the attributes that Fastly returns. I included every one; if there are any that are less interesting to you feel free to delete those. @@ -160,6 +171,7 @@ let batchAndSend = (aggregate, service) => { } sendToInsights = (logMessages) => { + const my_send_count = get_send_count() let insights_url = 'https://insights-collector.newrelic.com/v1/accounts/'+ ACCOUNT_ID +'/events' request({ method: "POST", @@ -170,7 +182,7 @@ sendToInsights = (logMessages) => { }, body: JSON.stringify(logMessages) }, (err, response, body) => { - console.log(body) - console.log(err) + console.log("Sent message (" + my_send_count + ") for " + logMessages.service_name + " at " + Date().toLocaleString() + ". Response: " + body) + if(err) console.log("Error: " + err); }) -} \ No newline at end of file +} From a2c817ef4681d0d4f30c5360776ba7eca098e076 Mon Sep 17 00:00:00 2001 From: Fastly2Git Date: Tue, 14 May 2019 09:24:54 -0600 Subject: [PATCH 3/4] no metadata for forked repo --- ww.metadata | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 ww.metadata diff --git a/ww.metadata b/ww.metadata deleted file mode 100644 index 8c4b84b..0000000 --- a/ww.metadata +++ /dev/null @@ -1,4 +0,0 @@ -#Metadata Labels -sec_repository=fastly-to-insights -sec_owner=parc -sec_critical_system=true From 82aa54a446346dc37ffd422205f5d19fd5282d45 Mon Sep 17 00:00:00 2001 From: Balazs Rauznitz Date: Tue, 14 May 2019 14:14:30 -0600 Subject: [PATCH 4/4] document new service_name option --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e32e6d..f5a1824 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The Fastly to Insights image is configured by environment variables. These are m * `INSERT_KEY` * `SERVICES` -`SERVICES` needs to be a string with the ids of the Fastly services you want to see data for in Insights, separated by a space. I know that's not ideal. A limitation of Fastly is that you have to query one service at a time, so I chose to create an array of service ids and loop through them to query Fastly. A limitation of Docker is that you can't pass an array via the command line, so I chose to split a string on "` `". If you have a better idea, I would love to hear it - please contribute! +`SERVICES` needs to be a string with the ids of the Fastly services, or SERVICE_NAME:SERVICE_ID pairs (Insights will receive `service_name`) you want to see data for in Insights, separated by a space. I know that's not ideal. A limitation of Fastly is that you have to query one service at a time, so I chose to create an array of service ids and loop through them to query Fastly. A limitation of Docker is that you can't pass an array via the command line, so I chose to split a string on "` `". If you have a better idea, I would love to hear it - please contribute! ### Example