Pythonifying the bash binaries (which should hopefully give a perform…

…ance boost

.github/workflows/ci-checks.yml

@@ -33,7 +33,15 @@ jobs:
         pip install -r requirements.txt -r requirements_dev.txt
     - name: Run black
       run: |
+        black --check bin/b64_decode
+        black --check bin/b64_encode
+        black --check bin/prepend_timestamp
+        black --check bin/raw_to_brefv
         black --check bin/to_csv
     - name: Run pylint
       run: |
+        pylint bin/b64_decode
+        pylint bin/b64_encode
+        pylint bin/prepend_timestamp
+        pylint bin/raw_to_brefv
         pylint bin/to_csv

bin/b64_decode

@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is base64 decoded.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is base64 decoded.
+"""
 
-set -euo pipefail
+import sys
+from base64 import b64decode
 
-while read -r line
-do
-  echo "$line" | base64 --decode
-done
+for line in sys.stdin:
+    sys.stdout.write(b64decode(line.encode()).decode())
+    sys.stdout.flush()

bin/b64_encode

@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is base64 encoded with no wrapping and ended with a newline.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is base64 encoded with no wrapping and ended with a newline.
+"""
 
-set -euo pipefail
+import sys
+from base64 import b64encode
 
-while read -r line
-do
-  echo "$line" | base64 --wrap=0; echo
-done
+for line in sys.stdin:
+    sys.stdout.write(b64encode(line.encode()).decode() + "\n")
+    sys.stdout.flush()

bin/prepend_timestamp

@@ -1,12 +1,14 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is prepended with the current timestamp in iso 8601 format according
-# to the RFC3339 subset.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is prepended with the current timestamp in iso 8601 format according
+to the RFC3339 subset.
+"""
 
-set -euo pipefail
+import sys
+from datetime import datetime, timezone
 
-while read -r line
-do
-  echo "$(date -u --rfc-3339=ns) $line"
-done
+for line in sys.stdin:
+    sys.stdout.write(f"{datetime.now(timezone.utc).isoformat()} {line}")
+    sys.stdout.flush()

bin/raw_to_brefv

@@ -1,12 +1,18 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is put in a separate brefv envelope (compact JSON output) with
-# the current timestamp in iso 8601 format according to the RFC3339 subset.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is put in a separate brefv envelope (compact JSON output) with
+the current timestamp in iso 8601 format according to the RFC3339 subset.
+"""
 
-set -euo pipefail
+import sys
+import json
+from datetime import datetime, timezone
 
-while read -r payload
-do
-    jo sent_at="$(date --rfc-3339=ns)" message="$payload" 
-done
+for line in sys.stdin:
+    output = json.dumps(
+        {"sent_at": datetime.now(timezone.utc).isoformat(), "message": line.strip()}
+    )
+    sys.stdout.write(output + "\n")
+    sys.stdout.flush()

bin/to_csv

@@ -37,7 +37,7 @@ def process(directory_path: Path, decode: bool = False):
     _timestamp_store = defaultdict(list)
     _payload_store = defaultdict(list)
 
-    for line in sys.stdin.readlines():
+    for line in sys.stdin:
         logged_at, topic, *payload = line.split(" ")
         payload = " ".join(payload)
         _timestamp_store[topic].append(logged_at)