Merge pull request #1 from akamai/development

Initial Commit for ULS development branch preparing v0.0.1

Dockerfile

@@ -0,0 +1,63 @@
+FROM            python:3.9.5-slim-buster
+LABEL 	        maintainer="Mike Schiessl - [email protected]"
+LABEL	        APP="Akamai Universal Log Streamer"
+
+# CONFIGURATION ARGS
+ARG             HOMEDIR="/opt/akamai-uls"
+ARG             ULS_DIR="$HOMEDIR/uls"
+ARG             EXT_DIR="$ULS_DIR/ext"
+
+ARG             ETP_CLI_VERSION="0.3.4"
+ARG             EAA_CLI_VERSION="0.3.8"
+ARG             MFA_CLI_VERSION="0.0.4"
+ARG             ULS_VERSION="0.0.1"
+
+# ENV VARS
+ENV             ULS_DIR=$ULS_DIR
+ENV             EXT_DIR=$EXT_DIR
+ENV             HOMEDIR=$HOMEDIR
+
+
+# PREPARE ENVIRONMENT
+# ENV PREP
+RUN	            apt-get update && \
+	            apt-get --no-install-recommends -y install \
+		        curl \
+		        ca-certificates \
+		        git && \
+		        rm -rf /var/lib/apt/lists/
+
+# USER & GROUP
+RUN 	        groupadd akamai && \
+                useradd -g akamai -s /bin/bash -m -d ${HOMEDIR} akamai
+
+USER            akamai
+WORKDIR         ${HOMEDIR}
+RUN             mkdir -p ${HOMEDIR}/uls
+
+
+# Install ULS
+ENV             ULS_VERSION=$ULS_VERSION
+RUN             git clone --depth 1 -b "${ULS_VERSION}" --single-branch https://github.com/akamai/uls.git ${ULS_DIR}
+WORKDIR         ${ULS_DIR}
+
+# Install external CLI'S
+## ETP CLI
+ENV             ETP_CLI_VERSION=$ETP_CLI_VERSION
+RUN             git clone --depth 1 -b "${ETP_CLI_VERSION}" --single-branch https://github.com/akamai/cli-etp.git ${EXT_DIR}/cli-etp && \
+                pip install -r ${EXT_DIR}/cli-etp/requirements.txt
+
+## EAA CLI
+ENV             EAA-CLI_VERSION=$EAA_CLI_VERSION
+RUN             git clone --depth 1 -b "${EAA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-eaa.git ${EXT_DIR}/cli-eaa && \
+                pip install -r ${EXT_DIR}/cli-eaa/requirements.txt
+## MFA CLI
+ENV             MFA-CLI_VERSION=$MFA_CLI_VERSION
+RUN             git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \
+                pip install -r ${EXT_DIR}/cli-mfa/requirements.txt
+
+# ENTRYPOINTS / CMD
+#CMD             /usr/local/bin/python3 ${ULS_DIR}/bin/uls.py
+ENTRYPOINT      ["/usr/local/bin/python3","bin/uls.py"]
+
+# EOF

README.md

@@ -1,2 +1,86 @@
-# uls
-Unified Log Streamer (ULS)
+# Akamai Unified Log Streamer (ULS)
+
+## Introduction
+The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Akamai Secure Enterprise Access Products
+- [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) 
+- [Enterprise Threat Protector (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
+- [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
+
+Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box.  
+It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts.
+
+
+![ULS docker compose usage](docs/images/ula_docker-compose_complex_example.png)
+
+
+## Table of contents
+- [Akamai Unified Log Streamer (ULS)](#akamai-unified-log-streamer-uls)
+  - [Introduction](#introduction)
+  - [Table of contents](#table-of-contents)
+  - [Key Features](#key-features)
+  - [Documentation](#documentation)
+    - [Command Line Usage](#command-line-usage)
+    - [Docker](#docker)
+    - [Docker-compose](#docker-compose)
+  - [Development](#development)
+  - [Support](#support)
+
+## Key Features
+
+- Supported Inputs (Secure Enterprise Access Products)
+    - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp)
+    - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
+    - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
+
+
+- Supported data outputs
+    - TCP Socket (tcp://host:port)
+    - UDP Socket (udp://host:port)
+    - HTTP(S) URL (http(s)://host:port/path) (supporting Authentication)
+
+
+- Operation types
+    - [python (command line)](./docs/COMMAND_LINE_USAGE.md)
+    - [docker](./docs/DOCKER_USAGE.md)
+    - [docker-compose](./docs/DOCKER-COMPOSE_USAGE.md)
+
+
+- Additional Features
+    - [Monitoring output](./docs/MONITORING.md)
+    - Debug information (log level adjustment)
+    - HTTP CA CERT verification skipping
+    - Adoptable HTTP - POST format
+
+## Documentation
+ULS can be operated in many ways.  
+Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM.
+More information for specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation.
+
+### Command Line Usage
+![ULS command line usage](docs/images/uls_cli_help_example.png)  
+For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md)
+
+### Docker
+![ULS docker usage](docs/images/uls_docker_etp_threat_example.png)  
+For more information, please visit [this document](./docs/DOCKER_USAGE.md)
+
+### Docker-compose
+![ULS docker compose usage](docs/images/ula_docker-compose_complex_example.png)  
+For more information, please visit [this document](./docs/DOCKER-COMPOSE_USAGE.md)
+
+
+## Development
+
+For the latest stable version of this software, please check the [release section](https://github.com/akamai/uls/releases) of the repo. The `main` [branch](https://github.com/akamai/uls) will retain the stable versions.
+To ensure a continuous development of this tool, all new updates will go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo.  
+The `development` branch can be subject to change and could also represent a broken version of this software.
+In parallel, all new versions within the "main" branch will also be available on the [ULS docker hub space](https://hub.docker.com/repository/docker/akamai/uls).
+
+Contributions to this software can be provided via [Pull Requests](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) and will get merged after successful review. 
+
+## Support
+
+Akamai ULS is provided "as-is". It is not supported by Akamai Support. Akamai is neither liable for the function nor for any caused problems that come along with the usage or caused by this tool. To report an issue, feature request or bug, please open a new issue into the [GitHub Issues page](https://github.com/akamai/uls/issues).
+This software is released under the "Apache License". Please refer to the [LICENSE](./LICENSE) document for more information.
+
+[Pull requests](#development) to improve the code or enhance the functionality are welcome.

bin/config/global_config.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# Common global variables / constants
+__version__ = "0.0.1"
+__tool_name_long__ = "Akamai Unified Log Streamer"
+__tool_name_short__ = "ULS"
+
+
+# Generic config
+bin_python = "python3"                          # Python binary to use (use OS standard when not using path)
+    # EAA
+bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"      # Path to the EAA CLI Executable
+eaa_cli_feeds = ['ACCESS', 'ADMIN']             # Available EAA CLI feeds
+    # ETP
+bin_etp_cli = "ext/cli-etp/bin/akamai-etp"      # Path to the ETP CLI Executable
+etp_cli_feeds = ['THREAT', 'AUP']               # Available ETP CLI feeds
+    # MFA
+bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa"      # Path to the MFA CLI Executable
+mfa_cli_feeds = ['POLICY', 'AUTH']              # Available MFA CLI feeds
+
+    # INPUT Choices
+input_choices = ['EAA', 'ETP', 'MFA']           # Available input types
+input_format_choices = ['JSON', 'TEXT']         # Available input format choices (need to be supported by cli)
+
+    # OUTPUT Choices
+output_choices = ['TCP', 'HTTP', 'UDP']         # Definition of OUTPUT Choices
+
+    # LogLevels
+log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+log_level_default = 'WARNING'
+
+
+# INPUT Configuration
+input_rerun_retries = 3                         # Number of rerun attempts before giving up
+input_run_delay = 1                             # Time in seconds to wait for the first health check
+input_rerun_delay = 1                           # Time in seconds between rerun attempts
+
+
+# OUTPUT Configuration
+output_reconnect_retries = 10                   # Number of reconnect attempts before giving up
+output_reconnect_delay = 1                      # Time in seconds between reconnect attempts
+output_udp_send_buffer = 262144                 # UDP Send buffer in bytes
+output_udp_timeout = 10.0                       # UDP SEND / CONNECT Timeout (seconds)
+output_tcp_send_buffer = 262144                 # TCP Send buffer in bytes
+output_tcp_timeout = 10.0                       # TCP SEND / CONNECT Timeout (seconds)
+                                                # Additional Headers to send (requests module KV pairs)
+output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'}
+
+
+# Monitoring Configuration
+monitoring_enabled = True                       # Set to false to disable monitoring outputs
+monitoring_interval = 5 * 60                    # Monitoring output interval (seconds)

bin/modules/UlsArgsParser.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import config.global_config as uls_config
+
+
+def init():
+    # Argument Parsing
+    parser = argparse.ArgumentParser(description=f"{uls_config.__tool_name_long__}",
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    # Common params
+    parser.add_argument('-l', '--loglevel',
+                        action='store',
+                        type=str.upper,
+                        default=(os.environ.get('ULS_LOGLEVEL') or uls_config.log_level_default),
+                        choices=uls_config.log_levels_available,
+                        help=f'Adjust the loglevel Default: {uls_config.log_level_default}')
+
+    # Version Information
+    parser.add_argument('-v', '--version',
+                        action='store',
+                        type=bool,
+                        default=False,
+                        nargs='?',
+                        const=True,
+                        help=f'Display {uls_config.__tool_name_short__} version and operational information')
+
+    # ----------------------
+    # Input GROUP
+    input_group = parser.add_argument_group(title="Input",
+                                            description="Define INPUT Settings (AKAMAI API)")
+
+    # INPUT_SELECTOR
+    input_group.add_argument('-i', '--input',
+                             action='store',
+                             type=str.upper,
+                             default=(os.environ.get('ULS_INPUT') or None),
+                             choices=uls_config.input_choices,
+                             help="Select the Input Source. Default: None", )
+    # INPUT_FEED
+    input_group.add_argument('--feed',
+                             action='store',
+                             type=str.upper,
+                             default=(os.environ.get('ULS_FEED') or 'DEFAULT'),
+                             help="Select data feed [CLI-DEFAULT]")
+    # INPUT FORMAT
+    input_group.add_argument('--format',
+                             action='store',
+                             dest="cliformat",
+                             type=str.upper,
+                             default=(os.environ.get('ULS_FORMAT') or "JSON"),
+                             choices=uls_config.input_format_choices,
+                             help="Select log output format Default: JSON")
+    # INPUT PROXY
+    input_group.add_argument('--inproxy', '--inputproxy',
+                             dest='inproxy',
+                             type=str,
+                             default=(os.environ.get('ULS_INPUT_PROXY') or None),
+                             help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)")
+    # RAWCMD
+    input_group.add_argument('--rawcmd',
+                             action='store',
+                             type=str,
+                             default=(os.environ.get('ULS_RAWCMD') or None),
+                             help="Overwrite the cli command with your parameters. (python3 akamai-cli $rawcmd)")
+    # EDGERC
+    input_group.add_argument('--edgerc',
+                             action='store',
+                             type=str,
+                             dest="credentials_file",
+                             default=(os.environ.get('ULS_EDGERC') or '~/.edgerc'),
+                             help="Location of the credentials file (default is ~/.edgerc)")
+    # EDGERC-SECTION
+    input_group.add_argument('--section',
+                             action='store',
+                             type=str,
+                             dest="credentials_file_section",
+                             default=(os.environ.get('ULS_SECTION') or 'default'),
+                             help="Credentials file Section's name to use ('default' if not specified).")
+
+    # ----------------------
+    # Output GROUP
+    output_group = parser.add_argument_group(title="Output",
+                                             description="Define OUTPUT Settings (SIEM)")
+
+    # OUTPUT Selector
+    output_group.add_argument('-o', '--output',
+                              action='store',
+                              type=str.upper,
+                              default=(os.environ.get('ULS_OUTPUT') or None),
+                              choices=uls_config.output_choices,
+                              help="Select the Output Destination Default: None")
+
+    # Output HOST
+    output_group.add_argument('--host',
+                              action='store',
+                              type=str,
+                              default=(os.environ.get('ULS_OUTPUT_HOST') or None),
+                              help="Host for TCP/UDP")
+
+    # OUTPUT PORT
+    output_group.add_argument('--port',
+                              action='store',
+                              type=int,
+                              default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'),
+                              help="Port for TCP/UDP")
+
+    # HTTP URL
+    output_group.add_argument('--httpurl',
+                              action='store',
+                              type=str,
+                              default=(os.environ.get('ULS_HTTP_URL') or None),
+                              help=f'Full http(s) target url i.e. '
+                                   f'https://my.splunk.host:9091/services/collector/event"')
+
+    # HTTP AUTH HEADER
+    output_group.add_argument('--httpauthheader',
+                              action='store',
+                              type=str,
+                              default=(os.environ.get('ULS_HTTP_AUTH_HEADER') or None),
+                              help='HTTP Header for authorization. Example: '
+                                   '\'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}\'')
+
+    # Disable HTTP TLS verification
+    output_group.add_argument('--httpinsecure',
+                              action='store',
+                              type=bool,
+                              default=(os.environ.get('ULS_HTTP_NO_VERIFY_TLS') or False),
+                              nargs='?',
+                              const=True,
+                              help=f'Disable TLS CA Certificate verification. Default: False')
+
+    # HTTP FORMAT DEFINITION
+    output_group.add_argument('--httpformat',
+                              action='store',
+                              type=str,
+                              default=(os.environ.get('ULS_HTTP_FORMAT') or '{"event": %s}'),
+                              help='HTTP Message format expected by http receiver '
+                                   '(%%s defines the data string). Default \'{\"event\": %%s}\'')
+
+    return parser.parse_args()
+
+# EOF