From 8b398cbde46c3e863d08ca5c567bdce2c60c1d81 Mon Sep 17 00:00:00 2001
From: Manav Darji <manavdarji.india@gmail.com>
Date: Wed, 20 Sep 2023 15:13:20 +0530
Subject: [PATCH 1/3] use otelcol-contrib package for traces instead of docker

---
 .gitignore                                |  2 +-
 configs/devnet/install-otelcol-contrib.sh | 47 +++++++++++++++++
 configs/devnet/otel-config-dd.yaml        | 54 +++++---------------
 src/express/commands/setup-datadog.js     | 62 +++++++++++++++--------
 src/express/commands/start.js             |  3 ++
 5 files changed, 107 insertions(+), 61 deletions(-)
 create mode 100644 configs/devnet/install-otelcol-contrib.sh

diff --git a/.gitignore b/.gitignore
index 3b71c9a9..9a96144f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
 node_modules
 test
 .DS_Store
-devnet
+/devnet
 .env
 *.tfvars
 .terraform*
diff --git a/configs/devnet/install-otelcol-contrib.sh b/configs/devnet/install-otelcol-contrib.sh
new file mode 100644
index 00000000..4e2167f4
--- /dev/null
+++ b/configs/devnet/install-otelcol-contrib.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+{ # Prevent execution if this script was only partially downloaded
+set -e
+
+oops() {
+    echo "$0:" "$@" >&2
+    exit 1
+}
+
+umask 0022
+
+tmpDir="$(mktemp -d -t otel-collector-contrib-XXXXXXXXXXX || \
+          oops "Can't create temporary directory for downloading files")"
+cleanup() {
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT INT QUIT TERM
+
+otel_version="0.85.0"
+
+baseUrl="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${otel_version}"
+binary="otelcol-contrib_${otel_version}_linux_amd64.deb"
+
+url="${baseUrl}/${binary}"
+package=$tmpDir/$binary
+
+if command -v curl > /dev/null 2>&1; then
+    fetch() { curl -L "$1" -o "$2"; }
+elif command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
+else
+    oops "you don't have wget or curl installed, which I need to download the binary package"
+fi
+
+echo "Downloading otelcol-contrib binary package from '$url' to '$tmpDir'..."
+fetch "$url" "$package" || oops "failed to download '$url'"
+
+echo "Removing existing installation..."
+sudo dpkg -r otelcol-contrib
+
+echo "Installing $package ..."
+sudo dpkg -i $package
+
+echo "open collector contrib has been installed successfully!"
+
+} # End of wrapping
diff --git a/configs/devnet/otel-config-dd.yaml b/configs/devnet/otel-config-dd.yaml
index 5dcb738a..efa34e5e 100644
--- a/configs/devnet/otel-config-dd.yaml
+++ b/configs/devnet/otel-config-dd.yaml
@@ -3,34 +3,15 @@ receivers:
     protocols:
       grpc:
         endpoint: 127.0.0.1:4317
-  hostmetrics:
-    collection_interval: 10s
-    scrapers:
-      paging:
-        metrics:
-          system.paging.utilization:
-            enabled: true
-      cpu:
-        metrics:
-          system.cpu.utilization:
-            enabled: true
-      disk: null
-      filesystem:
-        metrics:
-          system.filesystem.utilization:
-            enabled: true
-      load: null
-      memory: null
-      network: null
-      processes: null
+  # Collect own metrics
   prometheus:
     config:
       scrape_configs:
-        - job_name: otelcol
-          scrape_interval: 10s
-          static_configs:
-            - targets:
-                - 0.0.0.0:8888
+      - job_name: 'otel-collector'
+        scrape_interval: 10s
+        static_configs:
+        - targets: ['0.0.0.0:8888']
+
 processors:
   batch:
     timeout: 10s
@@ -38,21 +19,14 @@ exporters:
   datadog:
     api:
       key: ${DD_API_KEY}
+
 service:
   pipelines:
-    metrics:
-      receivers:
-        - hostmetrics
-        - otlp
-        - prometheus
-      processors:
-        - batch
-      exporters:
-        - datadog
     traces:
-      receivers:
-        - otlp
-      processors:
-        - batch
-      exporters:
-        - datadog
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [datadog]
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [batch]
+      exporters: [datadog]
diff --git a/src/express/commands/setup-datadog.js b/src/express/commands/setup-datadog.js
index d0e1f69a..064fd348 100644
--- a/src/express/commands/setup-datadog.js
+++ b/src/express/commands/setup-datadog.js
@@ -71,45 +71,67 @@ export async function setupDatadog() {
       envName = `devnet-${x}`
     }
 
+    // Setup datadog-agent
     console.log('📍Setting up datadog for', envName)
     let command = `DD_API_KEY=${apiKey} DD_SITE="datadoghq.com" DD_HOST_TAGS="env:${envName}" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)"`
     await runSshCommand(`${user}@${host}`, command, maxRetries)
     console.log(`📍Datadog installed on ${host}`)
 
-    await installDocker(`${user}@${host}`, user)
-    console.log('📍Docker installed')
+    // Copy the DD config
+    console.log('📍Copying the datadog config')
+    let src = './openmetrics-conf.yaml'
+    let dest = `${user}@${host}:~/conf.yaml`
+    await runScpCommand(src, dest, maxRetries)
+
+    command =
+      'sudo mv ~/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml'
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Restart the datadog-agent
+    console.log('📍Restarting the datadog agent')
+    command = 'sudo service datadog-agent restart'
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Setup otel collector service
+    console.log('📍Setting up otel collector for', envName)
+    src = './install-otelcol-contrib.sh'
+    dest = `${user}@${host}:~/install-otelcol-contrib.sh`
+    await runScpCommand(src, dest, maxRetries)
 
-    const datadogConfig = await yaml.load(
+    // Install otel collector using the script
+    command = "sudo bash ~/install-otelcol-contrib.sh"
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Set the datadog api key in the otel config
+    console.log('📍Setting up datadog api key in otel config and copying the config')
+    const otelConfig = await yaml.load(
       fs.readFileSync('./otel-config-dd.yaml', 'utf8'),
       undefined
     )
-    await setDatadogAPIKey(apiKey, datadogConfig)
+    await setDatadogAPIKey(apiKey, otelConfig)
 
-    let src = './otel-config-dd.yaml'
-    let dest = `${user}@${host}:~/otel-config-dd.yaml`
+    // Copy the otel config
+    src = './otel-config-dd.yaml'
+    dest = `${user}@${host}:~/otel-config-dd.yaml`
     await runScpCommand(src, dest, maxRetries)
 
-    src = './openmetrics-conf.yaml'
-    dest = `${user}@${host}:~/conf.yaml`
-    await runScpCommand(src, dest, maxRetries)
-
-    command =
-      'sudo mv ~/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml'
+    command = "sudo mv ~/otel-config-dd.yaml /etc/otelcol-contrib/config.yaml"
     await runSshCommand(`${user}@${host}`, command, maxRetries)
-
-    command =
-      'sudo docker run -d --net=host -v ~/otel-config-dd.yaml:/otel-local-config.yaml otel/opentelemetry-collector-contrib --config otel-local-config.yaml'
+    
+    // Restart the otel service
+    console.log('📍Restarting the otel service')
+    command = "sudo systemctl daemon-reload && sudo service otelcol-contrib restart"
     await runSshCommand(`${user}@${host}`, command, maxRetries)
-    console.log(`📍OpenCollector started on ${host}`)
 
-    command = 'sudo service datadog-agent restart'
+    // Remove the installation script
+    command = "rm ~/install-otelcol-contrib.sh"
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
     // revert dd api key
     // eslint-disable-next-line no-undef, no-template-curly-in-string
-    await setDatadogAPIKey('${DD_API_KEY}', datadogConfig)
+    await setDatadogAPIKey('${DD_API_KEY}', otelConfig)
   }
 
-  console.log('📍Datadog devnet env : ', envName)
-  console.log('📍Datadog setup complete')
+  console.log('📍Datadog devnet env:', envName)
+  console.log('📍Datadog and otel collector setup complete')
 }
diff --git a/src/express/commands/start.js b/src/express/commands/start.js
index c0ff4460..e5508493 100644
--- a/src/express/commands/start.js
+++ b/src/express/commands/start.js
@@ -488,6 +488,9 @@ export async function start() {
   shell.exec(
     `cp ../../configs/devnet/openmetrics-conf.yaml ../../deployments/devnet-${devnetId}`
   )
+  shell.exec(
+    `cp ../../configs/devnet/install-otelcol-contrib.sh ../../deployments/devnet-${devnetId}`
+  )
   shell.exec(
     `cp ../../configs/devnet/otel-config-dd.yaml ../../deployments/devnet-${devnetId}`
   )

From e32bf4c05fc67b730d51c0fa4ff7de9de960f248 Mon Sep 17 00:00:00 2001
From: Manav Darji <manavdarji.india@gmail.com>
Date: Wed, 20 Sep 2023 15:24:10 +0530
Subject: [PATCH 2/3] prettier fix

---
 src/express/commands/setup-datadog.js | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/express/commands/setup-datadog.js b/src/express/commands/setup-datadog.js
index 064fd348..e2db8605 100644
--- a/src/express/commands/setup-datadog.js
+++ b/src/express/commands/setup-datadog.js
@@ -99,11 +99,13 @@ export async function setupDatadog() {
     await runScpCommand(src, dest, maxRetries)
 
     // Install otel collector using the script
-    command = "sudo bash ~/install-otelcol-contrib.sh"
+    command = 'sudo bash ~/install-otelcol-contrib.sh'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
     // Set the datadog api key in the otel config
-    console.log('📍Setting up datadog api key in otel config and copying the config')
+    console.log(
+      '📍Setting up datadog api key in otel config and copying the config'
+    )
     const otelConfig = await yaml.load(
       fs.readFileSync('./otel-config-dd.yaml', 'utf8'),
       undefined
@@ -115,16 +117,17 @@ export async function setupDatadog() {
     dest = `${user}@${host}:~/otel-config-dd.yaml`
     await runScpCommand(src, dest, maxRetries)
 
-    command = "sudo mv ~/otel-config-dd.yaml /etc/otelcol-contrib/config.yaml"
+    command = 'sudo mv ~/otel-config-dd.yaml /etc/otelcol-contrib/config.yaml'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
-    
+
     // Restart the otel service
     console.log('📍Restarting the otel service')
-    command = "sudo systemctl daemon-reload && sudo service otelcol-contrib restart"
+    command =
+      'sudo systemctl daemon-reload && sudo service otelcol-contrib restart'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
     // Remove the installation script
-    command = "rm ~/install-otelcol-contrib.sh"
+    command = 'rm ~/install-otelcol-contrib.sh'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
     // revert dd api key

From 8ac4965e957ccd5f8e8348f3effcee669cfae35a Mon Sep 17 00:00:00 2001
From: Manav Darji <manavdarji.india@gmail.com>
Date: Wed, 20 Sep 2023 15:26:25 +0530
Subject: [PATCH 3/3] lint fix

---
 src/express/commands/setup-datadog.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/express/commands/setup-datadog.js b/src/express/commands/setup-datadog.js
index e2db8605..d68c487b 100644
--- a/src/express/commands/setup-datadog.js
+++ b/src/express/commands/setup-datadog.js
@@ -9,7 +9,6 @@ const {
   runSshCommand,
   maxRetries
 } = require('../common/remote-worker')
-const { installDocker } = require('./start.js')
 
 const yaml = require('js-yaml')
 const fs = require('fs')