Merge pull request #7 from akamai/development

release 0.0.4

Dockerfile

@@ -9,7 +9,7 @@ ARG             EXT_DIR="$ULS_DIR/ext"
 
 ARG             ETP_CLI_VERSION="0.3.5"
 ARG             EAA_CLI_VERSION="0.3.9"
-ARG             MFA_CLI_VERSION="0.0.4"
+ARG             MFA_CLI_VERSION="0.0.5"
 
 # ENV VARS
 ENV             ULS_DIR=$ULS_DIR

README.md

@@ -32,8 +32,15 @@ It can be run directly as Python code, as a provided Docker container or through
 
 - Supported Inputs (Secure Enterprise Access Products)
     - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp)
+      - ACCESS 
+      - ADMIN
+      - CONHEALTH
     - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
+      - THREAT
+      - AUP
     - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
+      - AUTH
+      - POLICY
 
 
 - Supported data outputs
@@ -60,8 +67,21 @@ ULS can be operated in many ways.
 Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM.
 More information for specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation.
 
+### Generic Requirements
+- Python 3.9+ OR docker / docker-compose 
+- AKAMAI .edgerc file
+- Internet access 
+
 ### Command Line Usage
 ![ULS command line usage](docs/images/uls_cli_help_example.png)  
+Example commands:
+```bash
+# ETP - THREAT to console
+python3.9 bin/uls.py --input etp --feed threat --output raw
+
+# EAA - ACCESS to TCP
+python3.0  bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081
+```
 For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md)
 
 ### Docker

bin/config/global_config.py

@@ -1,17 +1,22 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "0.0.3"
+__version__ = "0.0.4"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
 
 # Generic config
 bin_python = "python3"                          # Python binary to use (use OS standard when not using path)
 output_line_breaker = '\r\n'                    # Line breaking type (to split messages when streaming data)
+main_wait_default = 0.01                        # Default wait time within the main loop
+main_wait_max = 60                              # Maximum wait time for the main loop
+
     # EAA
-bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"      # Path to the EAA CLI Executable
-eaa_cli_feeds = ['ACCESS', 'ADMIN']             # Available EAA CLI feeds
+        # Path to the EAA CLI Executable
+bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"
+        # Available EAA CLI feeds
+eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH']
     # ETP
 bin_etp_cli = "ext/cli-etp/bin/akamai-etp"      # Path to the ETP CLI Executable
 etp_cli_feeds = ['THREAT', 'AUP']               # Available ETP CLI feeds

bin/modules/UlsArgsParser.py

@@ -52,7 +52,7 @@ def init():
                              choices=uls_config.input_choices,
                              help="Select the Input Source. Default: None", )
     # INPUT_FEED
-    input_group.add_argument('--feed',
+    input_group.add_argument('-f', '--feed',
                              action='store',
                              type=str.upper,
                              default=(os.environ.get('ULS_FEED') or 'DEFAULT'),

bin/modules/UlsInputCli.py

@@ -16,45 +16,14 @@
 import sys
 import time
 import shlex
-import platform
+
 import os
 
 # ULS modules
 import modules.aka_log as aka_log
 import config.global_config as uls_config
 
 
-def uls_version():
-    """
-    Collect ULS Version information and display it on STDOUT
-    """
-    def _get_cli_version(cli_bin):
-        try:
-            version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "version"],
-                                            stdout=subprocess.PIPE,
-                                            stderr=subprocess.PIPE)
-            my_cli_version = version_proc.communicate()[0].decode().strip('\n')
-            version_proc.terminate()
-            if my_cli_version:
-                return my_cli_version
-            else:
-                return "n/a"
-        except Exception as my_err:
-            return f"n/a -> ({my_err})"
-
-    # generate the stdout
-    print(f"{uls_config.__tool_name_long__} Version information\n"
-          f"ULS Version\t\t{uls_config.__version__}\n\n"
-          f"EAA Version\t\t{_get_cli_version(uls_config.bin_eaa_cli)}\n"
-          f"ETP Version\t\t{_get_cli_version(uls_config.bin_etp_cli)}\n"
-          f"MFA Version\t\t{_get_cli_version(uls_config.bin_mfa_cli)}\n\n"
-          f"OS Plattform\t\t{platform.platform()}\n"
-          f"OS Version\t\t{platform.release()}\n"
-          f"Python Version\t\t{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n"
-          )
-    sys.exit(0)
-
-
 class UlsInputCli:
     def __init__(self):
 
@@ -130,12 +99,15 @@ def proc_create(self, product=None,
                 product_feeds = uls_config.eaa_cli_feeds
                 if not rawcmd:
                     feed = self._feed_selector(feed, product_feeds)
-                    cli_command = [self.bin_python, product_path, 'log', feed.lower(), '-f']
+                    if feed == "CONHEALTH":
+                        cli_command = [self.bin_python, product_path, 'connector', 'list', '--perf', '--tail']
+                    else:
+                        cli_command = [self.bin_python, product_path, 'log', feed.lower(), '-f']
                     cli_command[2:2] = self._uls_useragent(product, feed)
                     cli_command[2:2] = edgegrid_auth
                     cli_command[2:2] = self._prep_proxy(inproxy)
                     if self._format_selector(cliformat) == "JSON":
-                        cli_command.append('-j')
+                        cli_command.append('--json')
                 else:
                     cli_command = [self.bin_python, product_path] + \
                                   self._uls_useragent(product, feed) +\
@@ -179,7 +151,7 @@ def proc_create(self, product=None,
                 aka_log.log.debug(f'{self.name} - CLI Command:  {cli_command}')
                 cli_proc = subprocess.Popen(cli_command,
                                             stdout=subprocess.PIPE,
-                                            stderr=subprocess.DEVNULL)
+                                            stderr=subprocess.PIPE)
 
                 aka_log.log.debug(f"{self.name} - started PID[{cli_proc.pid}]: {cli_command}")
                 self.proc = cli_proc
@@ -189,14 +161,17 @@ def proc_create(self, product=None,
 
                 if not self.check_proc():
                     raise NameError(f"process [{cli_proc.pid}] "
-                                    f"exited rc={cli_proc.returncode}: {cli_proc.stderr.read()}")
+                                    f"exited RC={cli_proc.returncode}, REASON: {cli_proc.stderr.read().decode()}")
+
+                # Handover the app into running state (disable stderr as it caused issues)
                 self.running = True
+                cli_proc.stderr = subprocess.DEVNULL
 
             except Exception as my_error:
                 time.sleep(self.rerun_delay)
                 self.running = False
                 rerun_counter += 1
-                aka_log.log.error(f'{self.name} - {my_error} -> {self.proc.stderr.read()}')
+                aka_log.log.error(f'{self.name} - {my_error} - {cli_proc.stderr.read().decode()}')
 
             if self.running is False and rerun_counter > self.rerun_retries:
                 aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors. '
@@ -206,11 +181,12 @@ def proc_create(self, product=None,
     def check_proc(self):
         try:
             if self.proc.poll() is None:
+
                 return True
             else:
                 self.running = False
                 aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]'
-                                  f' was found stale -> {self.proc.stderr.read()}')
+                                  f' was found stale - {cli_proc.stderr.read().decode()}')
                 return False
         except:
             return False

bin/modules/UlsOutput.py

@@ -206,6 +206,7 @@ def send_data(self, data):
             else:
                 aka_log.log.critical(f"{self.name} target was not defined {self.output_type} ")
                 sys.exit(1)
+
         except Exception as my_error:
             aka_log.log.error(f"{self.name} Issue sending data {my_error}")
             self.connected = False

bin/modules/UlsTools.py

@@ -0,0 +1,79 @@
+# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import sys
+import platform
+import os.path
+
+# ULS modules
+import modules.aka_log as aka_log
+import config.global_config as uls_config
+
+
+def uls_check_sys():
+    """
+    Collect ULS requirements information and request input if failing
+    """
+    def _check_cli_installed(cli_bin):
+        try:
+            if not os.path.isfile(cli_bin):
+                aka_log.log.warning(f"Uhoh - seems like {cli_bin} is not installed. "
+                                    f"Please follow docs/COMMAND_LINE_USAGE.md "
+                                    f"to setup the required environment cli tools")
+                skip_verification = input("Continue anyway ? (y|N)")
+                if skip_verification.lower() == "y" or skip_verification.lower() == "yes":
+                    print(f"Continuing without {cli_bin} - please be do not use any stream this cli provides")
+                else:
+                    aka_log.log.critical(f"Missing {cli_bin} - exiting")
+                    sys.exit(1)
+            else:
+                return True
+        except Exception as my_error:
+            aka_log.log.critical(f"Error checking the cli'tools ")
+
+    _check_cli_installed(uls_config.bin_eaa_cli)
+    _check_cli_installed(uls_config.bin_etp_cli)
+    _check_cli_installed(uls_config.bin_mfa_cli)
+
+
+def uls_version():
+    """
+    Collect ULS Version information and display it on STDOUT
+    """
+    def _get_cli_version(cli_bin):
+        try:
+            version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "version"],
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            my_cli_version = version_proc.communicate()[0].decode().strip('\n')
+            version_proc.terminate()
+            if my_cli_version:
+                return my_cli_version
+            else:
+                return "n/a"
+        except Exception as my_err:
+            return f"n/a -> ({my_err})"
+
+    # generate the stdout
+    print(f"{uls_config.__tool_name_long__} Version information\n"
+          f"ULS Version\t\t{uls_config.__version__}\n\n"
+          f"EAA Version\t\t{_get_cli_version(uls_config.bin_eaa_cli)}\n"
+          f"ETP Version\t\t{_get_cli_version(uls_config.bin_etp_cli)}\n"
+          f"MFA Version\t\t{_get_cli_version(uls_config.bin_mfa_cli)}\n\n"
+          f"OS Plattform\t\t{platform.platform()}\n"
+          f"OS Version\t\t{platform.release()}\n"
+          f"Python Version\t\t{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n"
+          )
+    sys.exit(0)

bin/uls.py

@@ -25,6 +25,7 @@
 import modules.UlsInputCli as UlsInputCli
 import config.global_config as uls_config
 import modules.UlsMonitoring as UlsMonitoring
+import modules.UlsTools as UlsTools
 
 stopEvent = threading.Event()
 
@@ -51,14 +52,20 @@ def main():
 
     signal.signal(signal.SIGTERM, sigterm_handler)
 
+
     # Load the Argument / ENV Var handler
     uls_args = aka_parser.init()
-    if uls_args.version:
-        UlsInputCli.uls_version()
 
     # Load the LOG system
     aka_log.init(uls_args.loglevel, uls_config.__tool_name_short__)
 
+    # Check CLI Environment
+    UlsTools.uls_check_sys()
+
+    # OUTPUT Version Information
+    if uls_args.version:
+        UlsTools.uls_version()
+
     # Create instances for Input and Output stream handler
     my_monitor = UlsMonitoring.UlsMonitoring(stopEvent, uls_args.input, uls_args.feed, uls_args.output)
     my_monitor.start()
@@ -67,8 +74,8 @@ def main():
 
     # When reading CLI output, if no data, pause 10ms before retrying
     # if still no data, then it will backoff exponentially till 60s
-    wait_default = 0.01
-    wait_max = 60
+    wait_default = uls_config.main_wait_default
+    wait_max = uls_config.main_wait_max
     wait = wait_default
 
     # Now let's handle the data and send input to output

docs/ARGUMENTS_ENV_VARS.md

@@ -14,7 +14,7 @@ The following tables list all available command line parameters and their corres
 |Parameter|Env - Var|Options|Default|Description|
 |---|---|---|---|---|
 |-i <br> --input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source |
-|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN'<br> ETP: 'THREAT', 'AUP'<br> MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed |
+|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'<br> ETP: 'THREAT', 'AUP'<br> MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed |
 |--format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format |
 |--inproxy <br> --inputproxy | ULS_INPUT_PROXY | HOST:PORT| None | Adjust proxy usage for INPUT data collection (cli) |
 |--rawcmd | ULS_RAWCMD | \<cli command\> | None | USE with caution /!\ <br> This is meant only to be used when told by AKAMAI|

docs/CHANGELOG.md

@@ -1,4 +1,20 @@
 # Version History
+## v0.0.4
+|||
+|---|---|
+|Date|2021-06-17
+|Kind|Bugfix / Feature
+|Author|[email protected]
+- Minor improvements 
+  - Wait_time and wait_max shifted to config
+  - added -f flag as alternative to --flag
+  - fixed an exception that was introduced in v0.0.3
+  - bumped MFA -CLI to 0.0.5 in dockerfile
+  - added an additional debugging example
+- Feature:
+  - EAA CONNECTOR HEALTH (CONHEALTH) now available
+  - Preflight (forced) check for available cli's
+
 ## v0.0.3
 |||
 |---|---|
@@ -11,7 +27,6 @@
 - bumped Dockerfile to newer CLI versions
 - introduced RAW output (send data to stdout)
 
-
 ## v0.0.2
 |||
 |---|---|