From 9896496183ce3d8624958a4bae48599dae83d51d Mon Sep 17 00:00:00 2001
From: Mike Schiessl <77062930+MikeSchiessl@users.noreply.github.com>
Date: Tue, 10 Oct 2023 12:19:17 +0200
Subject: [PATCH] v1.7.0 (#48)
preparing v1.7.0 release
---
Dockerfile | 8 +-
README.md | 3 +-
bin/config/global_config.py | 10 +-
bin/modules/UlsArgsParser.py | 18 ++
bin/modules/UlsInputCli.py | 10 +
bin/modules/UlsOutput.py | 23 ++-
bin/uls.py | 3 +-
docs/AKAMAI_API_CREDENTIALS.md | 8 +-
docs/ARGUMENTS_ENV_VARS.md | 6 +-
docs/CHANGELOG.md | 21 ++
docs/FAQ.md | 29 +++
docs/LOG_OVERVIEW.md | 188 +++++++++++++-----
docs/SIEM/SPLUNK/README.md | 35 +++-
.../kubernetes/helm/akamai-uls/Chart.yaml | 2 +-
scripts/get-uls.sh | 59 +++++-
test/positive_test.bats | 24 ++-
16 files changed, 375 insertions(+), 72 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 5a2f241..d086121 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.4-slim-bookworm
+FROM python:3.12.0-slim-bookworm
LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com"
LABEL APP_LONG="Akamai Universal Log Streamer"
LABEL APP_SHORT="ULS"
@@ -10,10 +10,10 @@ ARG HOMEDIR="/opt/akamai-uls"
ARG ULS_DIR="$HOMEDIR/uls"
ARG EXT_DIR="$ULS_DIR/ext"
-ARG ETP_CLI_VERSION="0.4.2"
-ARG EAA_CLI_VERSION="0.5.9"
+ARG ETP_CLI_VERSION="0.4.4"
+ARG EAA_CLI_VERSION="0.6.2"
ARG MFA_CLI_VERSION="0.1.1"
-ARG GC_CLI_VERSION="v0.0.1(beta)"
+ARG GC_CLI_VERSION="v0.0.2(beta)"
ARG LINODE_CLI_VERSION="dev"
# ENV VARS
diff --git a/README.md b/README.md
index 31b1880..3d5be42 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,7 @@ It can be run directly as Python code, as a provided Docker container, through
- [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin)
- [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth)
- [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv)
+ - [DIRHEALTH](docs/LOG_OVERVIEW.md#directory-health-dirhealth)
- [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
- [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat)
- [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup)
@@ -101,7 +102,7 @@ Example commands:
python3.9 bin/uls.py --input etp --feed threat --output raw
# EAA - ACCESS to TCP
-python3.9 bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081
+python3.9 bin/uls.py --input eaa --feed access --output tcp --host 10.99.10.99 --port 8081
```
For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md)
diff --git a/bin/config/global_config.py b/bin/config/global_config.py
index 8537944..06c6535 100644
--- a/bin/config/global_config.py
+++ b/bin/config/global_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# Common global variables / constants
-__version__ = "1.6.6"
+__version__ = "1.7.0"
__tool_name_long__ = "Akamai Unified Log Streamer"
__tool_name_short__ = "ULS"
@@ -19,7 +19,7 @@
# Path to the EAA CLI Executable
bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"
# Available EAA CLI feeds
-eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV']
+eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV', 'DIRHEALTH']
# ETP
# Path to the ETP CLI Executable
@@ -60,7 +60,7 @@
input_rerun_delay = 1 # Time in seconds between rerun attempts
input_disable_stderr = True # Enable STDERR output disabling (see value below to specify when this should happen)
input_disable_stderr_after = 25 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow
-input_queue_size = 10000 # Maximum number of events we want to store in-memory, default is 10000
+input_queue_size = 15000 # Maximum number of events we want to store in-memory, default is 10000
# OUTPUT Configuration
output_reconnect_retries = 10 # Number of reconnect attempts before giving up
@@ -77,6 +77,10 @@
output_http_aggregate_idle = 5 # Aggregate will send the data regardless of the count if the previous event was x secs ago
output_http_expected_status_code = 200 # Return Code for successful delivery
output_http_liveness_check = True # Send an OPTIONS request to probe the HTTP Server is live
+output_http_default_formattype = 'json-list' # The default "formattype" being used in standard operation
+output_http_formattypes = ['json-list', 'single-event'] # List of choices (valid formattypes)
+
+
## FILE
output_file_encoding = "utf-8" # FILE Encoding setting
output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler
diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py
index c9b9a04..6944c80 100644
--- a/bin/modules/UlsArgsParser.py
+++ b/bin/modules/UlsArgsParser.py
@@ -110,6 +110,14 @@ def init():
default=(os.environ.get('ULS_ENDTIME') or None),
help="End time (EPOCH SECONDS) until when to stop getting logs ('default': cli_default (never), example: '1631556101')")
+ # INPUT QUEUE SIZE
+ input_group.add_argument('--inputqueuesize',
+ action='store',
+ type=int,
+ dest="input_queue_size",
+ default=(os.environ.get('ULS_INPUT_QUEUESIZE') or uls_config.input_queue_size ),
+ help=f"Maximum threshold of the input queue. (Default: {uls_config.input_queue_size})")
+
# ----------------------
# Output GROUP
output_group = parser.add_argument_group(title="Output",
@@ -201,6 +209,16 @@ def init():
f"disable. Default: {uls_config.output_http_liveness_check}"
)
+ ## HTTP FORMATTYPE
+ output_group.add_argument('--httpformattype',
+ action='store',
+ type=str.lower,
+ default=(os.environ.get('ULS_HTTP_FORMAT_TYPE') or
+ uls_config.output_http_default_formattype),
+ choices=uls_config.output_http_formattypes,
+ help=f"Specifies the type how the given http format is being wrapped (controls, how the httpformat is being rendered in http output) "
+ f" Default: {uls_config.output_http_default_formattype}, Valid Choices: {uls_config.output_http_formattypes}")
+
# FILE STUFF
## File Handler
output_group.add_argument('--filehandler',
diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
index 0e38b11..7536e20 100644
--- a/bin/modules/UlsInputCli.py
+++ b/bin/modules/UlsInputCli.py
@@ -175,6 +175,16 @@ def proc_create(self):
'dp',
'inventory',
'--tail']
+ elif my_feed == "DIRHEALTH":
+ UlsTools.uls_check_edgerc(self.credentials_file,
+ self.credentials_file_section,
+ uls_config.edgerc_openapi)
+ cli_command = [self.bin_python,
+ '-u',
+ product_path,
+ 'dir',
+ 'list',
+ '--tail']
else:
UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
index 2eb96fc..21a78d1 100644
--- a/bin/modules/UlsOutput.py
+++ b/bin/modules/UlsOutput.py
@@ -47,6 +47,7 @@ def __init__(self, output_type: str,
http_url=None,
http_insecure=False,
http_liveness=True,
+ http_formattype=None,
filehandler=None,
filename=None,
filebackupcount=None,
@@ -116,6 +117,9 @@ def __init__(self, output_type: str,
self.http_out_aggregate_count = http_out_aggregate_count # Added for easier CLI configuration
self.aggregateListTick = None # Last time we added items in the list
# ---- End change for EME-588 ----
+ self.http_formattype = http_formattype
+
+
self.http_url = http_url
# apply other variables if SET
@@ -406,7 +410,6 @@ def send_data(self, data):
"""
try:
aka_log.log.debug(f"{self.name} Trying to send data via {self.output_type}")
-
if self.output_type == "TCP":
send_data = bytes(self.tcpudp_out_format, 'utf-8') % data
out_data = send_data + uls_config.output_line_breaker.encode()
@@ -429,7 +432,23 @@ def send_data(self, data):
self.aggregateListTick is not None and
self.aggregateListTick < time.time() - uls_config.output_http_aggregate_idle
):
- request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList)))
+
+
+ # JSON-LIST EVENT FORMAT: '{"event": [{logline1},{logline2},{logline3},{….},{logline500}]}'
+ # See https://github.com/akamai/uls/issues/45
+ if self.http_formattype.lower() == "json-list":
+ request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList)))
+
+ # Single EVENT FORMAT: '{"event": {logline1}}{"event": {logline2}}{"event": {….}}{"event": {logline500}}'
+ # See https://github.com/akamai/uls/issues/45
+ elif self.http_formattype.lower() == "single-event":
+ #[print(fruit + " juice") for fruit in fruits]
+ single_event_data = ""
+ for logline in self.aggregateList:
+ #print(f"logline: {self.http_out_format % logline}")
+ single_event_data = f"{single_event_data}{self.http_out_format % json.dumps(logline)}"
+ request = requests.Request('POST', url=self.http_url, data=(single_event_data))
+
prepped = self.httpSession.prepare_request(request)
payload_length = prepped.headers["Content-Length"]
diff --git a/bin/uls.py b/bin/uls.py
index 3146f84..824f874 100755
--- a/bin/uls.py
+++ b/bin/uls.py
@@ -121,6 +121,7 @@ def main():
http_url=uls_args.httpurl,
http_insecure=uls_args.httpinsecure,
http_liveness=uls_args.httpliveness,
+ http_formattype=uls_args.httpformattype,
filehandler=uls_args.filehandler,
filename=uls_args.filename,
filebackupcount=uls_args.filebackupcount,
@@ -162,7 +163,7 @@ def main():
# New ULS/1.5: the input module is ingesting messages
# into a thread safe queue. The function call will immediately
# return
- event_q = queue.Queue(uls_config.input_queue_size)
+ event_q = queue.Queue(uls_args.input_queue_size)
my_input.ingest(stopEvent, event_q, my_monitor)
# Now we are back to the main thread to process the message
diff --git a/docs/AKAMAI_API_CREDENTIALS.md b/docs/AKAMAI_API_CREDENTIALS.md
index 0cb3f5f..59a356f 100644
--- a/docs/AKAMAI_API_CREDENTIALS.md
+++ b/docs/AKAMAI_API_CREDENTIALS.md
@@ -16,10 +16,10 @@ This document describes how to create Akamai API credentials and configure them
- [Guardicore](#guardicore)
- [Guardicore API Integration](#guardicore-api-integration)
- [Linode](#linode)
- - [Linode API Token](#linode-api-credentials)
+ - [Linode API Credentials](#linode-api-credentials)
- [Advanced .edgerc usage](#advanced-edgerc-usage)
- [Multiple customer contracts](#multiple-customer-contracts)
- - [Partner & employee enhancement](#partner--employee-enhancement)
+ - [Partner \& employee enhancement](#partner--employee-enhancement)
- [ETP API EVENT Filters](#etp-api-event-filters)
## Feeds / API overview
@@ -27,8 +27,8 @@ This document describes how to create Akamai API credentials and configure them
|Product long name|Acronym| Feed(s) | API |
|---|---|---------------------------------|---------------------------------------------------------------------------------------|
|Enterprise Application Access|EAA| ACCESS, ADMIN | [EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds) |
-|Enterprise Application Access|EAA| HEALTH | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) |
-|Enterprise Threat Protector|ETP| THREAT, AUP, DNS, PROXY | [{OPEN} API / ETP Report](#etp-open-api-reporting) |
+|Enterprise Application Access|EAA| CONHEALTH, DEVINV, DIRHEALTH | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) |
+|Secure Internet Access Enterprise|ETP| THREAT, AUP, DNS, PROXY, NETCON | [{OPEN} API / ETP Report](#etp-open-api-reporting) |
|Akamai MFA|MFA| EVENTS | [MFA Integration](#mfa-integration-for-logging) |
|Guardicore|GC| NETLOG, INCIDENT, AGENT, SYSTEM | [Guardicore API Integration](#guardicore-api-integration) |
|Linode|LN| AUDIT | [Linode API Credentials](#linode-api-credentials) |
diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
index 3737025..4b82b34 100644
--- a/docs/ARGUMENTS_ENV_VARS.md
+++ b/docs/ARGUMENTS_ENV_VARS.md
@@ -23,13 +23,14 @@ The following tables list all available command line parameters and their corres
| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file |
| --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. |
| --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! |
+| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" |
## OUTPUT
| Parameter | Output Type | Env - Var | Options | Default | Description |
|------------------|-------------|----------------------|----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| -o
--output | | ULS_OUTPUT | 'TCP', 'UDP', 'HTTP', 'RAW', 'FILE' | None | Specify the desired OUTPUT target |
- | | | | | |
+ | | | | | |
| --host | TCP / UDP | ULS_OUTPUT_HOST | xxx.xxx.xxx.xxx | None | Specify the desired OUTPUT target host (TCP/UDP only) |
| --port | TCP / UDP | ULS_OUTPUT_PORT | xxxx | None | Specify the desired OUTPUT target port (TCP/UDP only) |
| --tcpudpformat | TCP / UDP | ULS_TCPUDP_FORMAT | '' | '%s' | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once |
@@ -38,8 +39,9 @@ The following tables list all available command line parameters and their corres
| --httpformat | HTTP(S) | ULS_HTTP_FORMAT | '' | '{"event": %s}' | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once |
| --httpauthheader | HTTP(S) | ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}' | None | Specify an Auhtorization header to auth against the HTTP Server (HTTP only)
Example:
'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}' |
| --httpinsecure | HTTP(S) | ULS_HTTP_INSECURE | True | False | Disable TLS CA certificate verification |
-| --httpliveness | HTTP(S) | ULS_HTTP_LIVENESS_CHECK | True, False | True | Perform liveness check with OPTIONS request that must return 200 or 204 if enabled|
+| --httpliveness | HTTP(S) | ULS_HTTP_LIVENESS_CHECK | True, False | True | Perform liveness check with OPTIONS request that must return 200 or 204 if enabled|
| --httpaggregate | HTTP(S) | ULS_HTTP_AGGREGATE | xxxx | 500 | Number of events to aggregate for one output request the %s in the httpformat will be replaced by a LIST of events.
A value of 1 means no aggregation.
Example: %s = [{'event1': 'data1'},{'event2': 'data2'},...] |
+| --httpformattype | HTTP(S) | ULS_HTTP_FORMAT_TYPE | 'JSON-LIST',SINGLE-EVENT' | 'JSON-LIST' | Specifies the type how the given http format is being wrapped (controls, how the httpformat is being rendered in http output) |
| | | | | | |
| --filehandler | FILE | ULS_FILE_HANDLER | 'SIZE','TIME' | SIZE | Select the handler which decides how the files are rotated if either specific SIZE or TIME has been reached |
| --filename | FILE | ULS_FILE_NAME | '/path/to/file.name' | None | The PATH + FILENAME where ULS should create the file |
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index cbbf185..a2bcbbb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,4 +1,25 @@
# Version History
+## v1.7.0
+|||
+|---|---|
+|Date|2023-10-10
+|Kind| FEATURE release
+|Author|mschiess@akamai.com, androcho@akamai.com
+- **Features**
+ - Allowing the configuration of the HTTPFORMATTYPE, which controls the building of payloads for aggregated HTTP requests (click [here](FAQ.md#what-is-http-formattype) for additional information)
+ - Allow adjustment of the "INPUT QUEUE SIZE" threshold (--inputqueuesize) in order to handle huge API pages and fast API output
+ - New feed for EAA: Directory Health (dirhealth) to fetch health details for configured directories wihtin EAA
+- **Minor improvements**
+ - Added additional checking in the auto installer
+ - [docker] bumped python version to "3.12.0"
+ - [docker] bumped GC-LOGS version to "0.0.2(beta)", now supporting credentials in ENV VARS
+ - [docker] bumped CLI-EAA to "0.6.2"
+ - [docker] bumped CLI-ETP version to "0.4.4" - fixed a bug in output ordering + empty response handling.
+ - "get_uls.sh" now allows selection of OS package installation rather than pip3. [See](https://github.com/akamai/uls/issues/46) for more information
+- **Housekeeping**
+ - DocFix Readme.md (thx [@ihommani](https://github.com/akamai/uls/pull/47))
+ - Increased default input_queue_size from 10000 to 15000 to avoid race conditions when an API is answering very fast
+
## v1.6.6
|||
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 148a95a..d4aa14a 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -12,6 +12,8 @@
- [ULS does not start due to missing field in config](#uls-does-not-start-due-to-missing-field-in-config)
- [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc)
- [WHY JMESPATH and not JSONPATH](#why-jmespath-and-not-jsonpath)
+- [What is HTTP FORMATTYPE](#what-is-http-formattype)
+-
----
## FAQ
@@ -145,3 +147,30 @@ JMESPATH has a very stable and [well defined language specification](https://jme
This gives a user way more options than "pure" jsonpath and is also the reason we decided to go along with the more flexible integration.
---
+### What is HTTP FORMATTYPE
+As some SIEM operate way more performant, when not having to parse JSON to separate log lines when receiving HTTP requests, ULS 1.6.7 introduces a way to actually control the behavior how data is sent within an HTTP request.
+While using the `--httpformattype` flag or the `ULS_HTTP_FORMAT_TYPE` ENV variable the following options can be choosen:
+#### JSON-LIST
+The HTTP paypload will be a concatenated list of log-lines which will replace the %s variable within the HTTP FORMAT.
+
+**Example:**
+HTTP_FORMAT: `'{"event": %s}'`
+Aggregated list: `[{logline1},{logline2},{logline3},{….},{logline500}]`
+Final Output Example: `'{"event": [{logline1},{logline2},{logline3},{….},{logline500}]}'`
+
+
+#### SINGLE-EVENT
+The HTTP payload will be a single HTTP FORMAT type filled with one logline but still containing multiple loglines per paypload
+
+**Example:**
+HTTP_FORMAT: `'{"event": %s}'`
+Aggregated list: `[{logline1},{logline2},{logline3},{….},{logline500}]`
+Final Output Example: '{"event": {logline1}}{"event": {logline2}}{"event": {….}}{"event": {logline500}}'
+
+Within the `single-event` mode, you can freely amend line breake configuration like `\n` or others, by amending it to the HTTP_FORMAT e.g. HTTP_FORMAT: `'{"event": %s}\n'`
+
+---
+### Error: "Capacity exceeded, too many incoming data vs. slow output"
+This error indicates, that more data is coming in to ULS than it can send towards the sepcified output.
+As this might be an indication for I/O problems either on the ULS output or the receiving system, it could also just be a specific race condition when the API operations with big pages or at a high speed (e.g. within local LAN).
+If requried, the size can be adjusted by using the "--inputqueuesize" introduced in ULS 1.6.7.
\ No newline at end of file
diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md
index c988c72..5f10e64 100644
--- a/docs/LOG_OVERVIEW.md
+++ b/docs/LOG_OVERVIEW.md
@@ -6,6 +6,7 @@ To get the highest value out of the ingested data, it is crucial to understand t
Here are some examples (per product) and links to additional information.
## Table of contents
+
- [Log Overview](#log-overview)
- [Table of contents](#table-of-contents)
- [Enterprise Application Access (EAA)](#enterprise-application-access-eaa)
@@ -13,10 +14,11 @@ Here are some examples (per product) and links to additional information.
- [Admin Logs (ADMIN)](#admin-logs-admin)
- [Connector Health (CONHEALTH)](#connector-health-conhealth)
- [Device Posture Inventory (DEVINV)](#device-posture-inventory-devinv)
- - [Enterprise Threat Protector (ETP)](#enterprise-threat-protector-etp)
+ - [Directory Health (DIRHEALTH)](#directory-health-dirhealth)
+ - [Secure Internet Access Enterprise (SIA-E)](#secure-internet-access-enterprise-sia-e)
- [Threat Log (THREAT)](#threat-log-threat)
- [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup)
- - [DNS](#dns)
+ - [DNS Activity](#dns-activity)
- [PROXY](#proxy)
- [NETCON](#netcon)
- [Akamai MFA (MFA)](#akamai-mfa-mfa)
@@ -24,9 +26,8 @@ Here are some examples (per product) and links to additional information.
- [Guardicore](#guardicore)
- [NETLOG](#netlog)
- [INCIDENT](#incident)
- - AGENT
- - SYSTEM
- - Linode
+ - [Linode](#linode)
+ - [AUDIT Logs](#audit-logs)
## Enterprise Application Access (EAA)
@@ -97,6 +98,7 @@ Additional information regarding the log fields can be found on [here](https://t
"privateip": "10.1.4.206",
"publicip": "123.123.123.123",
"debugchan": "Y",
+ "datetime": "2021-07-23T18:06:35.676Z",
"ts": "2021-07-23T18:06:35.676Z",
"cpu": 1.3,
"disk": 34.4,
@@ -251,10 +253,51 @@ Each event will be one device as a JSON document, example provided with the cli-
```
-## Enterprise Threat Protector (ETP)
+### Directory Health (DIRHEALTH)
+
+Each event will be one directory as a JSON document.
+Examples provided can be obtained using cli-eaa command `akamai eaa dir list --json|jq .`
+Schema is documented on the [EAA Directory List API doc](https://techdocs.akamai.com/eaa-api/reference/get-directories).
+
+
+ View directory health event example (JSON)
+
+```json
+{
+ "dir_id": "dir://49L59MSsQcyeaRz6N8iKmA",
+ "service": "ActiveDirectory",
+ "name": "gc-eaa-forrestor-ActiveDirectory",
+ "datetime": "2023-10-06T22:02:00.112396+00:00",
+ "status": 1,
+ "connector_count": 1,
+ "directory_status": "ok",
+ "group_count": 3,
+ "user_count": 8,
+ "last_sync": "2023-10-06T15:55:31.026068",
+ "sync_state": "Dirty",
+ "conf_state": 1
+}
+```
+
+
+## Secure Internet Access Enterprise (SIA-E)
+
+Formerly known as Enterprise Threat Protector (ETP).
+
+For large volume of security events (multiple 100K per hour), configure the underlying
+`cli-etp` to issue concurrent API requests.
+
+Depending on your ULS setup you need to pass the `CLIETP_FETCH_CONCURRENT` environment variable.
+We recommend to start with the value `2`, observe, and increase up to `8` if you observe backlog.
+
+This will have a small impact on CPU usage, while increasing the number of events.
### Threat Log (THREAT)
Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-threat-event-details)
+
+
+ Security Threat Event example (JSON)
+
```json
{
"pageInfo": {
@@ -643,9 +686,13 @@ Additional information regarding the log fields can be found [here](https://tech
]
}
```
+
### Accceptable Use Policy Logs (AUP)
Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/get-events-details)
+
+
+ Acceptable Use Policy Event example (JSON)
```json
{
"pageInfo": {
@@ -1034,9 +1081,13 @@ Additional information regarding the log fields can be found [here](https://tech
]
}
```
+
-### DNS
+### DNS Activity
Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-dns-activities-details)
+
+
+ DNS Activity Event example (JSON)
```json
{
"pageInfo": {
@@ -1261,9 +1312,14 @@ Additional information regarding the log fields can be found [here](https://tech
]
}
```
+
### PROXY
+
Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-traffic-transaction-details)
+
+
+ Proxy Activity Event example (JSON)
```json
{
"pageInfo": {
@@ -2086,12 +2142,54 @@ Additional information regarding the log fields can be found [here](https://tech
]
}
```
+
### NETCON
Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-network-traffic-connections-details)
+
+
+ Network Connection Event example (JSON)
```json
-{"id": "123", "connectionId": "0xABCDEF1234567890", "domain": "123.123.123.123", "connStartTime": "2023-08-23T07:59:11Z", "connEndTime": "2023-08-23T07:59:11Z", "clientIP": "222.111.222.111", "clientPort": 35593, "destinationIP": "111.222.111.222", "destinationPort": 80, "siteId": 1234536, "siteName": "ETP DEMO", "policyAction": "onramp", "onrampType": "explicit_proxy_tls", "internalClientIP": "", "httpVersion": "N/A", "httpUserAgent": "", "machineId": "", "machineName": "", "clientRequestId": "", "ovfActionId": -1, "ovfActionName": "N/A", "stats": {"httpRequestCount": 1, "inBytes": 0, "outBytes": 0}, "dropInfo": {"wasDropped": true, "droppedReason": "Destination Filter - Internal Host IP"}, "encryptedInternalClientIP": "123123123123123123/ABCDEF", "decryptedInternalClientIP": "192.168.11.168", "sublocationId": "-1", "sublocationName": "N/A", "deviceOwnerId": "", "encryptedInternalClientName": ""}
+{
+ "id": "123",
+ "connectionId": "0xABCDEF1234567890",
+ "domain": "123.123.123.123",
+ "connStartTime": "2023-08-23T07:59:11Z",
+ "connEndTime": "2023-08-23T07:59:11Z",
+ "clientIP": "222.111.222.111",
+ "clientPort": 35593,
+ "destinationIP": "111.222.111.222",
+ "destinationPort": 80,
+ "siteId": 1234536,
+ "siteName": "ETP DEMO",
+ "policyAction": "onramp",
+ "onrampType": "explicit_proxy_tls",
+ "internalClientIP": "",
+ "httpVersion": "N/A",
+ "httpUserAgent": "",
+ "machineId": "",
+ "machineName": "",
+ "clientRequestId": "",
+ "ovfActionId": -1,
+ "ovfActionName": "N/A",
+ "stats": {
+ "httpRequestCount": 1,
+ "inBytes": 0,
+ "outBytes": 0
+ },
+ "dropInfo": {
+ "wasDropped": true,
+ "droppedReason": "Destination Filter - Internal Host IP"
+ },
+ "encryptedInternalClientIP": "123123123123123123/ABCDEF",
+ "decryptedInternalClientIP": "192.168.11.168",
+ "sublocationId": "-1",
+ "sublocationName": "N/A",
+ "deviceOwnerId": "",
+ "encryptedInternalClientName": ""
+}
```
+
## Akamai MFA (MFA)
Additional information regarding the MFA log fields can be found on [here](https://techdocs.akamai.com/mfa/docs/splunk-app).
@@ -2136,44 +2234,42 @@ Authentication Events Example:
Guardicore netlog example
```json
{
-'flow_id':'XXXXX2045a3630852de54dcb99e86f6b06d3969050e153efaed1cb2c4',
-'bucket_id':'XXXX-62fa-459a-80e1-c96c2eacdee9',
-'source_node_id':'XXXX-3a21-4508-87bd-d0a6512442bc',
-'destination_node_id':'UnknownAsset_Internal_192.168.0.0/16',
-'source_node_type':'asset',
-'destination_node_type':'subnet',
-'source_process':'gc-channel',
-'destination_process':'Unknown Server (443/TCP)',
-'source_process_id':'XXXX491e858806fc17a722c7e93f780e867df4800e6a9bddcc396abf39250',
-'destination_process_id':'7XXXX5aad6d81ebe5037013fded72223e12e2d8a0d4e4823c90232139b',
-'source_process_name':'gc-channel',
-'destination_process_name':'Unknown Server (443/TCP)',
-'destination_port':443,
-'count':2,
-'slot_start_time':XXX72413000,
-'incidents':False,
-'connection_type':'UNKNOWN',
-'source_ip':'192.168.2.76',
-'destination_ip':'192.168.2.68',
-'ip_protocol':'Tcp',
-'source_asset_hash':317458,
-'destination_asset_hash':349875,
-'violates_policy':False,
-'policy_rule':'default',
-'policy_ruleset':None,
-'policy_verdict':'allowed',
-'db_insert_time':'XXXX-11-09T04:09:54.293504',
-'id':'XXXXXX-7d27-48b0-ab66-cdedcbc444c3',
-'source':{
-'vm':{
-'_id':'XXXXXX-3a21-4508-87bd-d0a6512442bc',
-'name':'Gollum Lab Server'
-}
-},
-'has_mismatch_alert':False,
-'original_policy_verdict':'allowed',
-'source_process_full_path':'/var/lib/guardicore/sbin/gc-channel',
-'destination_process_full_path':None
+ "id": "123",
+ "connectionId": "0xABCDEF1234567890",
+ "domain": "123.123.123.123",
+ "connStartTime": "2023-08-23T07:59:11Z",
+ "connEndTime": "2023-08-23T07:59:11Z",
+ "clientIP": "222.111.222.111",
+ "clientPort": 35593,
+ "destinationIP": "111.222.111.222",
+ "destinationPort": 80,
+ "siteId": 1234536,
+ "siteName": "ETP DEMO",
+ "policyAction": "onramp",
+ "onrampType": "explicit_proxy_tls",
+ "internalClientIP": "",
+ "httpVersion": "N/A",
+ "httpUserAgent": "",
+ "machineId": "",
+ "machineName": "",
+ "clientRequestId": "",
+ "ovfActionId": -1,
+ "ovfActionName": "N/A",
+ "stats": {
+ "httpRequestCount": 1,
+ "inBytes": 0,
+ "outBytes": 0
+ },
+ "dropInfo": {
+ "wasDropped": true,
+ "droppedReason": "Destination Filter - Internal Host IP"
+ },
+ "encryptedInternalClientIP": "123123123123123123/ABCDEF",
+ "decryptedInternalClientIP": "192.168.11.168",
+ "sublocationId": "-1",
+ "sublocationName": "N/A",
+ "deviceOwnerId": "",
+ "encryptedInternalClientName": ""
}
```
diff --git a/docs/SIEM/SPLUNK/README.md b/docs/SIEM/SPLUNK/README.md
index 555cf79..c40b92c 100644
--- a/docs/SIEM/SPLUNK/README.md
+++ b/docs/SIEM/SPLUNK/README.md
@@ -3,6 +3,7 @@
## Table of contents
- [Introduction](#introduction)
+- [Configuration Options](#configuration-options)
- [Additional Splunk Documentation](#additional-splunk-documentation)
- [Key Performance Indicators for EAA](#key-performance-indicators-for-eaa)
- [[1] No dialout (per connector)](#1-no-dialout-per-connector)
@@ -32,12 +33,44 @@ index=akamai source=uls_etp_threat | spath | top event.actionName
Splunk also works perfectly with the ULS provided [monitoring data](../../MONITORING.md)
+## Configuration Options
+### HTTP
+Splunk ingestion via HTTP towards the SPLUNK HEC can be done via multiple ways:
+- LIST of JSON LOG EVENTS
+ This method required JSON parsing on the INPUT - minimizing the HTTP payload size. Payload example:
+ ```json
+ {"event": [{logevent1},{logevent2},{...}]}
+ ```
+ The required ULS settings look as follows:
+ ```bash
+ bin/uls.py \
+ -i eaa \
+ -f access \
+ -o http \
+ --httpurl https://splunk-URL:8088/services/collector/event \
+ --httpauthheader '{"Authorization": "Splunk 123-321-456"}'
+ ```
+- SINGLE EVENT
+ This method allows a raw ingestion of the data, increasing the HTTP payload size. Payload example:
+ ```json
+ {"event": {logevent1}}{"event": {logevent2}}{"event": {...}}
+ ```
+ The required ULS settings look as follows:
+ ```bash
+ bin/uls.py \
+ -i eaa \
+ -f access \
+ -o http \
+ --httpurl https://splunk-URL:8088/services/collector/event \
+ --httpauthheader '{"Authorization": "Splunk 123-321-456"}' \
+ --httpformattype SINGLE-EVENT
+ ```
+
## Additional Splunk Documentation
- [TCP](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/Monitornetworkports)
- [HTTP](https://docs.splunk.com/Documentation/Splunk/8.2.0/Data/UsetheHTTPEventCollector)
- [UDP](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/Monitornetworkports)
-
## Key Performance Indicators for EAA
Splunk search queries provided as examples, feel free to customize them as you see fit.
diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
index da260b7..0447870 100644
--- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
+++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation
type: application
version: 2.0.0
-appVersion: "1.6.6"
+appVersion: "1.7.0"
diff --git a/scripts/get-uls.sh b/scripts/get-uls.sh
index cb4fa76..2b8ed58 100755
--- a/scripts/get-uls.sh
+++ b/scripts/get-uls.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# This file will install the latest ULS including all of its modules (latest version) into the current directory/uls
-# bash $(curl https://)
+# curl -O https://raw.githubusercontent.com/akamai/uls/main/scripts/get-uls.sh && bash get-uls.sh
default_modules="eaa,etp,mfa,gc,ln"
default_install_dir="$(pwd)/uls"
@@ -25,23 +25,44 @@ if [[ -z $(which git) ]] ; then
fi
## check python version
+### check python exists
if [[ -z $(which python3) ]] ; then
echo "Python3 binary was not found - exiting"
exit 1
fi
+### check python version is correct for ULS
py_version=$(python3 --version | cut -d " " -f 2)
if [[ $(echo ${py_version} | cut -d "." -f 1 ) -ne 3 ]] || [[ $(echo ${py_version} | cut -d "." -f 2 ) -lt 9 ]]; then
echo "Wrong Python version - exiting"
exit 1
fi
+
## pip3
if [[ -z $(which pip3) ]] ; then
echo "pip3 binary was not found - exiting"
exit 1
fi
+### Show versions to verify the correct binaries for python and pip are being used
+echo "We will use the following python binaries to install ULS:"
+echo -ne "python3: \t $(ls $(which python3))\n"
+echo -ne "pip3: \t\t $(ls $(which pip3))\n\n"
+echo -ne "Is this correct (Y|n)"
+read py_reply
+
+
+case $py_reply in
+ n|N)
+ echo -e "Not the right version ?\nPlease adjust your ENV and SYMLINK settings to point to the correct binaries."
+ echo -e "EXITING !"
+ exit 1
+ ;;
+ *)
+ echo "Continuing"
+esac
+
# ASK for INSTALL DETAILS
## Install DIR
echo -n "Installation Dir [Default: \"$default_install_dir\"] (ENTER for default or new value): "
@@ -83,19 +104,43 @@ if [[ -f "${install_dir}/bin/uls.py" ]] ; then
fi
fi
-## Continue anywaY ?
-
-
-# Installation
+## Continue anyway ?
+# Installation
## Grab ULS
git clone -q https://github.com/akamai/uls.git $install_dir/
pip3 install -q -r ${install_dir}/bin/requirements.txt
+function py_reqs() {
+ to_install=$(pip3 install --dry-run -r $1 | grep -vi "satisfied")
+ if [[ ! -z $to_install ]] ; then
+ echo "We are going to install the following python3 requirements:"
+ echo "----"
+ echo $to_install
+ echo "----"
+
+ echo "Do you want to stop here and install OS packages instead [y|N]?"
+ read package_stop
+ case $package_stop in
+ y|Y)
+ echo "Stopping installation - please install the requried packages manually - exiting"
+ exit 1
+ ;;
+ *)
+ echo "installing packages via PIP now"
+ ;;
+ esac
+ fi
+
+
+}
+
## Grab EAA-CLI
if [[ "$install_modules" == *"eaa"* ]] ; then
echo "Installing EAA-CLI"
git clone -q --depth 1 --single-branch https://github.com/akamai/cli-eaa.git ${install_dir}/ext/cli-eaa
+ #echo "${install_dir}/ext/cli-eaa/requirements.txt"
+ py_reqs ${install_dir}/ext/cli-eaa/requirements.txt
pip3 install -q -r ${install_dir}/ext/cli-eaa/requirements.txt
fi
@@ -103,6 +148,7 @@ fi
if [[ "$install_modules" == *"etp"* ]] ; then
echo "Installing ETP-CLI"
git clone -q --depth 1 --single-branch https://github.com/akamai/cli-etp.git ${install_dir}/ext/cli-etp
+ py_reqs ${install_dir}/ext/cli-etp/requirements.txt
pip3 install -q -r ${install_dir}/ext/cli-etp/requirements.txt
fi
@@ -110,6 +156,7 @@ fi
if [[ "$install_modules" == *"mfa"* ]] ; then
echo "Installing MFA-CLI"
git clone -q --depth 1 --single-branch https://github.com/akamai/cli-mfa.git ${install_dir}/ext/cli-mfa
+ py_reqs ${install_dir}/ext/cli-mfa/requirements.txt
pip3 install -q -r ${install_dir}/ext/cli-mfa/requirements.txt
fi
@@ -117,6 +164,7 @@ fi
if [[ "$install_modules" == *"gc"* ]] ; then
echo "Installing GC-CLI"
git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/gc-logs.git ${install_dir}/ext/cli-gc
+ py_reqs ${install_dir}/ext/cli-gc/bin/requirements.txt
pip3 install -q -r ${install_dir}/ext/cli-gc/bin/requirements.txt
fi
@@ -125,6 +173,7 @@ fi
if [[ "$install_modules" == *"ln"* ]] ; then
echo "Installing LINODE-CLI"
git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/ln-logs.git ${install_dir}/ext/cli-linode
+ py_reqs ${install_dir}/ext/cli-linode/bin/requirements.txt
pip3 install -q -r ${install_dir}/ext/cli-linode/bin/requirements.txt
fi
diff --git a/test/positive_test.bats b/test/positive_test.bats
index 751bafc..49a9040 100644
--- a/test/positive_test.bats
+++ b/test/positive_test.bats
@@ -11,9 +11,9 @@ mocked_edgerc=FALSE
# TIMEOUT
# How much time is timeout alklowed to run
- uls_test_timeout=20
+ uls_test_timeout=90
# Send a kill signal after
- uls_kill_timeout=30
+ uls_kill_timeout=110
# Used for regular timeout
uls_timeout_signal="TERM"
uls_timeout_params=" --preserve-status --kill-after $uls_kill_timeout --signal ${uls_timeout_signal} ${uls_test_timeout} "
@@ -97,6 +97,16 @@ load 'bats/bats-assert/load.bash'
#[ "$status" -eq 124 ] #return value from timeout without --preserve status
[ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout
}
+@test "EAA - DIRHEALTH" {
+ run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed dirhealth --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info
+ #assert_output --partial $eaa_devinv_assert
+ assert_line --partial "UlsInputCli - started PID"
+ refute_line --partial "was found stale -"
+ #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting"
+ #[ "$status" -eq 124 ] #return value from timeout without --preserve status
+ [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout
+}
+
## ETP
@test "ETP - THREAT" {
@@ -139,6 +149,16 @@ load 'bats/bats-assert/load.bash'
[ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout
}
+@test "ETP - NETCON" {
+ run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed netcon --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info
+ #assert_output --partial $etp_assert
+ assert_line --partial "UlsInputCli - started PID"
+ refute_line --partial "was found stale -"
+ #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting"
+ #[ "$status" -eq 124 ] #return value from timeout without --preserve status
+ [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout
+}
+
## MFA
@test "MFA - EVENT" {
run timeout ${uls_timeout_params} ${uls_bin} --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info