Use otelcol-contrib package for traces instead of docker

.gitignore

@@ -1,7 +1,7 @@
 node_modules
 test
 .DS_Store
-devnet
+/devnet
 .env
 *.tfvars
 .terraform*

configs/devnet/install-otelcol-contrib.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+{ # Prevent execution if this script was only partially downloaded
+set -e
+
+oops() {
+    echo "$0:" "$@" >&2
+    exit 1
+}
+
+umask 0022
+
+tmpDir="$(mktemp -d -t otel-collector-contrib-XXXXXXXXXXX || \
+          oops "Can't create temporary directory for downloading files")"
+cleanup() {
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT INT QUIT TERM
+
+otel_version="0.85.0"
+
+baseUrl="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${otel_version}"
+binary="otelcol-contrib_${otel_version}_linux_amd64.deb"
+
+url="${baseUrl}/${binary}"
+package=$tmpDir/$binary
+
+if command -v curl > /dev/null 2>&1; then
+    fetch() { curl -L "$1" -o "$2"; }
+elif command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
+else
+    oops "you don't have wget or curl installed, which I need to download the binary package"
+fi
+
+echo "Downloading otelcol-contrib binary package from '$url' to '$tmpDir'..."
+fetch "$url" "$package" || oops "failed to download '$url'"
+
+echo "Removing existing installation..."
+sudo dpkg -r otelcol-contrib
+
+echo "Installing $package ..."
+sudo dpkg -i $package
+
+echo "open collector contrib has been installed successfully!"
+
+} # End of wrapping

configs/devnet/otel-config-dd.yaml

@@ -3,56 +3,30 @@ receivers:
     protocols:
       grpc:
         endpoint: 127.0.0.1:4317
-  hostmetrics:
-    collection_interval: 10s
-    scrapers:
-      paging:
-        metrics:
-          system.paging.utilization:
-            enabled: true
-      cpu:
-        metrics:
-          system.cpu.utilization:
-            enabled: true
-      disk: null
-      filesystem:
-        metrics:
-          system.filesystem.utilization:
-            enabled: true
-      load: null
-      memory: null
-      network: null
-      processes: null
+  # Collect own metrics
   prometheus:
     config:
       scrape_configs:
-        - job_name: otelcol
-          scrape_interval: 10s
-          static_configs:
-            - targets:
-                - 0.0.0.0:8888
+      - job_name: 'otel-collector'
+        scrape_interval: 10s
+        static_configs:
+        - targets: ['0.0.0.0:8888']
+
 processors:
   batch:
     timeout: 10s
 exporters:
   datadog:
     api:
       key: ${DD_API_KEY}
+
 service:
   pipelines:
-    metrics:
-      receivers:
-        - hostmetrics
-        - otlp
-        - prometheus
-      processors:
-        - batch
-      exporters:
-        - datadog
     traces:
-      receivers:
-        - otlp
-      processors:
-        - batch
-      exporters:
-        - datadog
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [datadog]
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [batch]
+      exporters: [datadog]

src/express/commands/setup-datadog.js

@@ -9,7 +9,6 @@ const {
   runSshCommand,
   maxRetries
 } = require('../common/remote-worker')
-const { installDocker } = require('./start.js')
 
 const yaml = require('js-yaml')
 const fs = require('fs')
@@ -71,45 +70,70 @@ export async function setupDatadog() {
       envName = `devnet-${x}`
     }
 
+    // Setup datadog-agent
     console.log('📍Setting up datadog for', envName)
     let command = `DD_API_KEY=${apiKey} DD_SITE="datadoghq.com" DD_HOST_TAGS="env:${envName}" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)"`
     await runSshCommand(`${user}@${host}`, command, maxRetries)
     console.log(`📍Datadog installed on ${host}`)
 
-    await installDocker(`${user}@${host}`, user)
-    console.log('📍Docker installed')
+    // Copy the DD config
+    console.log('📍Copying the datadog config')
+    let src = './openmetrics-conf.yaml'
+    let dest = `${user}@${host}:~/conf.yaml`
+    await runScpCommand(src, dest, maxRetries)
+
+    command =
+      'sudo mv ~/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml'
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Restart the datadog-agent
+    console.log('📍Restarting the datadog agent')
+    command = 'sudo service datadog-agent restart'
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Setup otel collector service
+    console.log('📍Setting up otel collector for', envName)
+    src = './install-otelcol-contrib.sh'
+    dest = `${user}@${host}:~/install-otelcol-contrib.sh`
+    await runScpCommand(src, dest, maxRetries)
 
-    const datadogConfig = await yaml.load(
+    // Install otel collector using the script
+    command = 'sudo bash ~/install-otelcol-contrib.sh'
+    await runSshCommand(`${user}@${host}`, command, maxRetries)
+
+    // Set the datadog api key in the otel config
+    console.log(
+      '📍Setting up datadog api key in otel config and copying the config'
+    )
+    const otelConfig = await yaml.load(
       fs.readFileSync('./otel-config-dd.yaml', 'utf8'),
       undefined
     )
-    await setDatadogAPIKey(apiKey, datadogConfig)
+    await setDatadogAPIKey(apiKey, otelConfig)
 
-    let src = './otel-config-dd.yaml'
-    let dest = `${user}@${host}:~/otel-config-dd.yaml`
+    // Copy the otel config
+    src = './otel-config-dd.yaml'
+    dest = `${user}@${host}:~/otel-config-dd.yaml`
     await runScpCommand(src, dest, maxRetries)
 
-    src = './openmetrics-conf.yaml'
-    dest = `${user}@${host}:~/conf.yaml`
-    await runScpCommand(src, dest, maxRetries)
-
-    command =
-      'sudo mv ~/conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml'
+    command = 'sudo mv ~/otel-config-dd.yaml /etc/otelcol-contrib/config.yaml'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
+    // Restart the otel service
+    console.log('📍Restarting the otel service')
     command =
-      'sudo docker run -d --net=host -v ~/otel-config-dd.yaml:/otel-local-config.yaml otel/opentelemetry-collector-contrib --config otel-local-config.yaml'
+      'sudo systemctl daemon-reload && sudo service otelcol-contrib restart'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
-    console.log(`📍OpenCollector started on ${host}`)
 
-    command = 'sudo service datadog-agent restart'
+    // Remove the installation script
+    command = 'rm ~/install-otelcol-contrib.sh'
     await runSshCommand(`${user}@${host}`, command, maxRetries)
 
     // revert dd api key
     // eslint-disable-next-line no-undef, no-template-curly-in-string
-    await setDatadogAPIKey('${DD_API_KEY}', datadogConfig)
+    await setDatadogAPIKey('${DD_API_KEY}', otelConfig)
   }
 
-  console.log('📍Datadog devnet env : ', envName)
-  console.log('📍Datadog setup complete')
+  console.log('📍Datadog devnet env:', envName)
+  console.log('📍Datadog and otel collector setup complete')
 }

src/express/commands/start.js

@@ -488,6 +488,9 @@ export async function start() {
   shell.exec(
     `cp ../../configs/devnet/openmetrics-conf.yaml ../../deployments/devnet-${devnetId}`
   )
+  shell.exec(
+    `cp ../../configs/devnet/install-otelcol-contrib.sh ../../deployments/devnet-${devnetId}`
+  )
   shell.exec(
     `cp ../../configs/devnet/otel-config-dd.yaml ../../deployments/devnet-${devnetId}`
   )