From ffbbaef2c0377204777c761d2aca43d7994c26be Mon Sep 17 00:00:00 2001
From: Fredrik Olsson <freol@outlook.com>
Date: Fri, 15 Apr 2022 17:41:45 +0200
Subject: [PATCH] Pythonifying the bash binaries (which should hopefully give a
 performance boost

---
 .github/workflows/ci-checks.yml |  8 ++++++++
 bin/b64_decode                  | 18 ++++++++++--------
 bin/b64_encode                  | 18 ++++++++++--------
 bin/prepend_timestamp           | 20 +++++++++++---------
 bin/raw_to_brefv                | 24 +++++++++++++++---------
 bin/to_csv                      |  2 +-
 6 files changed, 55 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml
index 7b5e552..93e6089 100644
--- a/.github/workflows/ci-checks.yml
+++ b/.github/workflows/ci-checks.yml
@@ -33,7 +33,15 @@ jobs:
         pip install -r requirements.txt -r requirements_dev.txt
     - name: Run black
       run: |
+        black --check bin/b64_decode
+        black --check bin/b64_encode
+        black --check bin/prepend_timestamp
+        black --check bin/raw_to_brefv
         black --check bin/to_csv
     - name: Run pylint
       run: |
+        pylint bin/b64_decode
+        pylint bin/b64_encode
+        pylint bin/prepend_timestamp
+        pylint bin/raw_to_brefv
         pylint bin/to_csv
diff --git a/bin/b64_decode b/bin/b64_decode
index 39b3771..cb5cf15 100644
--- a/bin/b64_decode
+++ b/bin/b64_decode
@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is base64 decoded.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is base64 decoded.
+"""
 
-set -euo pipefail
+import sys
+from base64 import b64decode
 
-while read -r line
-do
-  echo "$line" | base64 --decode
-done
\ No newline at end of file
+for line in sys.stdin:
+    sys.stdout.write(b64decode(line.encode()).decode())
+    sys.stdout.flush()
diff --git a/bin/b64_encode b/bin/b64_encode
index 394d953..2ce42d3 100644
--- a/bin/b64_encode
+++ b/bin/b64_encode
@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is base64 encoded with no wrapping and ended with a newline.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is base64 encoded with no wrapping and ended with a newline.
+"""
 
-set -euo pipefail
+import sys
+from base64 import b64encode
 
-while read -r line
-do
-  echo "$line" | base64 --wrap=0; echo
-done
\ No newline at end of file
+for line in sys.stdin:
+    sys.stdout.write(b64encode(line.encode()).decode() + "\n")
+    sys.stdout.flush()
diff --git a/bin/prepend_timestamp b/bin/prepend_timestamp
index 9910df3..e10565b 100644
--- a/bin/prepend_timestamp
+++ b/bin/prepend_timestamp
@@ -1,12 +1,14 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is prepended with the current timestamp in iso 8601 format according
-# to the RFC3339 subset.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is prepended with the current timestamp in iso 8601 format according
+to the RFC3339 subset.
+"""
 
-set -euo pipefail
+import sys
+from datetime import datetime, timezone
 
-while read -r line
-do
-  echo "$(date -u --rfc-3339=ns) $line"
-done
\ No newline at end of file
+for line in sys.stdin:
+    sys.stdout.write(f"{datetime.now(timezone.utc).isoformat()} {line}")
+    sys.stdout.flush()
diff --git a/bin/raw_to_brefv b/bin/raw_to_brefv
index 6a4a08f..5e2f1b7 100644
--- a/bin/raw_to_brefv
+++ b/bin/raw_to_brefv
@@ -1,12 +1,18 @@
-#!/bin/bash
+#!/usr/bin/env python3.9
 
-# Command line utility tool for processing input from stdin. Each line on the
-# input stream is put in a separate brefv envelope (compact JSON output) with
-# the current timestamp in iso 8601 format according to the RFC3339 subset.
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is put in a separate brefv envelope (compact JSON output) with
+the current timestamp in iso 8601 format according to the RFC3339 subset.
+"""
 
-set -euo pipefail
+import sys
+import json
+from datetime import datetime, timezone
 
-while read -r payload
-do
-    jo sent_at="$(date --rfc-3339=ns)" message="$payload" 
-done
\ No newline at end of file
+for line in sys.stdin:
+    output = json.dumps(
+        {"sent_at": datetime.now(timezone.utc).isoformat(), "message": line.strip()}
+    )
+    sys.stdout.write(output + "\n")
+    sys.stdout.flush()
diff --git a/bin/to_csv b/bin/to_csv
index a953cb7..ec61603 100644
--- a/bin/to_csv
+++ b/bin/to_csv
@@ -37,7 +37,7 @@ def process(directory_path: Path, decode: bool = False):
     _timestamp_store = defaultdict(list)
     _payload_store = defaultdict(list)
 
-    for line in sys.stdin.readlines():
+    for line in sys.stdin:
         logged_at, topic, *payload = line.split(" ")
         payload = " ".join(payload)
         _timestamp_store[topic].append(logged_at)