From 9896496183ce3d8624958a4bae48599dae83d51d Mon Sep 17 00:00:00 2001 From: Mike Schiessl <77062930+MikeSchiessl@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:19:17 +0200 Subject: [PATCH] v1.7.0 (#48) preparing v1.7.0 release --- Dockerfile | 8 +- README.md | 3 +- bin/config/global_config.py | 10 +- bin/modules/UlsArgsParser.py | 18 ++ bin/modules/UlsInputCli.py | 10 + bin/modules/UlsOutput.py | 23 ++- bin/uls.py | 3 +- docs/AKAMAI_API_CREDENTIALS.md | 8 +- docs/ARGUMENTS_ENV_VARS.md | 6 +- docs/CHANGELOG.md | 21 ++ docs/FAQ.md | 29 +++ docs/LOG_OVERVIEW.md | 188 +++++++++++++----- docs/SIEM/SPLUNK/README.md | 35 +++- .../kubernetes/helm/akamai-uls/Chart.yaml | 2 +- scripts/get-uls.sh | 59 +++++- test/positive_test.bats | 24 ++- 16 files changed, 375 insertions(+), 72 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5a2f241..d086121 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.4-slim-bookworm +FROM python:3.12.0-slim-bookworm LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com" LABEL APP_LONG="Akamai Universal Log Streamer" LABEL APP_SHORT="ULS" @@ -10,10 +10,10 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.4.2" -ARG EAA_CLI_VERSION="0.5.9" +ARG ETP_CLI_VERSION="0.4.4" +ARG EAA_CLI_VERSION="0.6.2" ARG MFA_CLI_VERSION="0.1.1" -ARG GC_CLI_VERSION="v0.0.1(beta)" +ARG GC_CLI_VERSION="v0.0.2(beta)" ARG LINODE_CLI_VERSION="dev" # ENV VARS diff --git a/README.md b/README.md index 31b1880..3d5be42 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ It can be run directly as Python code, as a provided Docker container, through - [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin) - [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth) - [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv) + - [DIRHEALTH](docs/LOG_OVERVIEW.md#directory-health-dirhealth) - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) - [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat) - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup) @@ -101,7 +102,7 @@ Example commands: python3.9 bin/uls.py --input etp --feed threat --output raw # EAA - ACCESS to TCP -python3.9 bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081 +python3.9 bin/uls.py --input eaa --feed access --output tcp --host 10.99.10.99 --port 8081 ``` For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md) diff --git a/bin/config/global_config.py b/bin/config/global_config.py index 8537944..06c6535 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.6.6" +__version__ = "1.7.0" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -19,7 +19,7 @@ # Path to the EAA CLI Executable bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa" # Available EAA CLI feeds -eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'] +eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV', 'DIRHEALTH'] # ETP # Path to the ETP CLI Executable @@ -60,7 +60,7 @@ input_rerun_delay = 1 # Time in seconds between rerun attempts input_disable_stderr = True # Enable STDERR output disabling (see value below to specify when this should happen) input_disable_stderr_after = 25 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow -input_queue_size = 10000 # Maximum number of events we want to store in-memory, default is 10000 +input_queue_size = 15000 # Maximum number of events we want to store in-memory, default is 10000 # OUTPUT Configuration output_reconnect_retries = 10 # Number of reconnect attempts before giving up @@ -77,6 +77,10 @@ output_http_aggregate_idle = 5 # Aggregate will send the data regardless of the count if the previous event was x secs ago output_http_expected_status_code = 200 # Return Code for successful delivery output_http_liveness_check = True # Send an OPTIONS request to probe the HTTP Server is live +output_http_default_formattype = 'json-list' # The default "formattype" being used in standard operation +output_http_formattypes = ['json-list', 'single-event'] # List of choices (valid formattypes) + + ## FILE output_file_encoding = "utf-8" # FILE Encoding setting output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index c9b9a04..6944c80 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -110,6 +110,14 @@ def init(): default=(os.environ.get('ULS_ENDTIME') or None), help="End time (EPOCH SECONDS) until when to stop getting logs ('default': cli_default (never), example: '1631556101')") + # INPUT QUEUE SIZE + input_group.add_argument('--inputqueuesize', + action='store', + type=int, + dest="input_queue_size", + default=(os.environ.get('ULS_INPUT_QUEUESIZE') or uls_config.input_queue_size ), + help=f"Maximum threshold of the input queue. (Default: {uls_config.input_queue_size})") + # ---------------------- # Output GROUP output_group = parser.add_argument_group(title="Output", @@ -201,6 +209,16 @@ def init(): f"disable. Default: {uls_config.output_http_liveness_check}" ) + ## HTTP FORMATTYPE + output_group.add_argument('--httpformattype', + action='store', + type=str.lower, + default=(os.environ.get('ULS_HTTP_FORMAT_TYPE') or + uls_config.output_http_default_formattype), + choices=uls_config.output_http_formattypes, + help=f"Specifies the type how the given http format is being wrapped (controls, how the httpformat is being rendered in http output) " + f" Default: {uls_config.output_http_default_formattype}, Valid Choices: {uls_config.output_http_formattypes}") + # FILE STUFF ## File Handler output_group.add_argument('--filehandler', diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 0e38b11..7536e20 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -175,6 +175,16 @@ def proc_create(self): 'dp', 'inventory', '--tail'] + elif my_feed == "DIRHEALTH": + UlsTools.uls_check_edgerc(self.credentials_file, + self.credentials_file_section, + uls_config.edgerc_openapi) + cli_command = [self.bin_python, + '-u', + product_path, + 'dir', + 'list', + '--tail'] else: UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py index 2eb96fc..21a78d1 100644 --- a/bin/modules/UlsOutput.py +++ b/bin/modules/UlsOutput.py @@ -47,6 +47,7 @@ def __init__(self, output_type: str, http_url=None, http_insecure=False, http_liveness=True, + http_formattype=None, filehandler=None, filename=None, filebackupcount=None, @@ -116,6 +117,9 @@ def __init__(self, output_type: str, self.http_out_aggregate_count = http_out_aggregate_count # Added for easier CLI configuration self.aggregateListTick = None # Last time we added items in the list # ---- End change for EME-588 ---- + self.http_formattype = http_formattype + + self.http_url = http_url # apply other variables if SET @@ -406,7 +410,6 @@ def send_data(self, data): """ try: aka_log.log.debug(f"{self.name} Trying to send data via {self.output_type}") - if self.output_type == "TCP": send_data = bytes(self.tcpudp_out_format, 'utf-8') % data out_data = send_data + uls_config.output_line_breaker.encode() @@ -429,7 +432,23 @@ def send_data(self, data): self.aggregateListTick is not None and self.aggregateListTick < time.time() - uls_config.output_http_aggregate_idle ): - request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList))) + + + # JSON-LIST EVENT FORMAT: '{"event": [{logline1},{logline2},{logline3},{….},{logline500}]}' + # See https://github.com/akamai/uls/issues/45 + if self.http_formattype.lower() == "json-list": + request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList))) + + # Single EVENT FORMAT: '{"event": {logline1}}{"event": {logline2}}{"event": {….}}{"event": {logline500}}' + # See https://github.com/akamai/uls/issues/45 + elif self.http_formattype.lower() == "single-event": + #[print(fruit + " juice") for fruit in fruits] + single_event_data = "" + for logline in self.aggregateList: + #print(f"logline: {self.http_out_format % logline}") + single_event_data = f"{single_event_data}{self.http_out_format % json.dumps(logline)}" + request = requests.Request('POST', url=self.http_url, data=(single_event_data)) + prepped = self.httpSession.prepare_request(request) payload_length = prepped.headers["Content-Length"] diff --git a/bin/uls.py b/bin/uls.py index 3146f84..824f874 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -121,6 +121,7 @@ def main(): http_url=uls_args.httpurl, http_insecure=uls_args.httpinsecure, http_liveness=uls_args.httpliveness, + http_formattype=uls_args.httpformattype, filehandler=uls_args.filehandler, filename=uls_args.filename, filebackupcount=uls_args.filebackupcount, @@ -162,7 +163,7 @@ def main(): # New ULS/1.5: the input module is ingesting messages # into a thread safe queue. The function call will immediately # return - event_q = queue.Queue(uls_config.input_queue_size) + event_q = queue.Queue(uls_args.input_queue_size) my_input.ingest(stopEvent, event_q, my_monitor) # Now we are back to the main thread to process the message diff --git a/docs/AKAMAI_API_CREDENTIALS.md b/docs/AKAMAI_API_CREDENTIALS.md index 0cb3f5f..59a356f 100644 --- a/docs/AKAMAI_API_CREDENTIALS.md +++ b/docs/AKAMAI_API_CREDENTIALS.md @@ -16,10 +16,10 @@ This document describes how to create Akamai API credentials and configure them - [Guardicore](#guardicore) - [Guardicore API Integration](#guardicore-api-integration) - [Linode](#linode) - - [Linode API Token](#linode-api-credentials) + - [Linode API Credentials](#linode-api-credentials) - [Advanced .edgerc usage](#advanced-edgerc-usage) - [Multiple customer contracts](#multiple-customer-contracts) - - [Partner & employee enhancement](#partner--employee-enhancement) + - [Partner \& employee enhancement](#partner--employee-enhancement) - [ETP API EVENT Filters](#etp-api-event-filters) ## Feeds / API overview @@ -27,8 +27,8 @@ This document describes how to create Akamai API credentials and configure them |Product long name|Acronym| Feed(s) | API | |---|---|---------------------------------|---------------------------------------------------------------------------------------| |Enterprise Application Access|EAA| ACCESS, ADMIN | [EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds) | -|Enterprise Application Access|EAA| HEALTH | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) | -|Enterprise Threat Protector|ETP| THREAT, AUP, DNS, PROXY | [{OPEN} API / ETP Report](#etp-open-api-reporting) | +|Enterprise Application Access|EAA| CONHEALTH, DEVINV, DIRHEALTH | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) | +|Secure Internet Access Enterprise|ETP| THREAT, AUP, DNS, PROXY, NETCON | [{OPEN} API / ETP Report](#etp-open-api-reporting) | |Akamai MFA|MFA| EVENTS | [MFA Integration](#mfa-integration-for-logging) | |Guardicore|GC| NETLOG, INCIDENT, AGENT, SYSTEM | [Guardicore API Integration](#guardicore-api-integration) | |Linode|LN| AUDIT | [Linode API Credentials](#linode-api-credentials) | diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index 3737025..4b82b34 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -23,13 +23,14 @@ The following tables list all available command line parameters and their corres | --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | | --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | | --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" | ## OUTPUT | Parameter | Output Type | Env - Var | Options | Default | Description | |------------------|-------------|----------------------|----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | -o
--output | | ULS_OUTPUT | 'TCP', 'UDP', 'HTTP', 'RAW', 'FILE' | None | Specify the desired OUTPUT target | - | | | | | | + | | | | | | | --host | TCP / UDP | ULS_OUTPUT_HOST | xxx.xxx.xxx.xxx | None | Specify the desired OUTPUT target host (TCP/UDP only) | | --port | TCP / UDP | ULS_OUTPUT_PORT | xxxx | None | Specify the desired OUTPUT target port (TCP/UDP only) | | --tcpudpformat | TCP / UDP | ULS_TCPUDP_FORMAT | '' | '%s' | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once | @@ -38,8 +39,9 @@ The following tables list all available command line parameters and their corres | --httpformat | HTTP(S) | ULS_HTTP_FORMAT | '' | '{"event": %s}' | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once | | --httpauthheader | HTTP(S) | ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}' | None | Specify an Auhtorization header to auth against the HTTP Server (HTTP only)
Example:
'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}' | | --httpinsecure | HTTP(S) | ULS_HTTP_INSECURE | True | False | Disable TLS CA certificate verification | -| --httpliveness | HTTP(S) | ULS_HTTP_LIVENESS_CHECK | True, False | True | Perform liveness check with OPTIONS request that must return 200 or 204 if enabled| +| --httpliveness | HTTP(S) | ULS_HTTP_LIVENESS_CHECK | True, False | True | Perform liveness check with OPTIONS request that must return 200 or 204 if enabled| | --httpaggregate | HTTP(S) | ULS_HTTP_AGGREGATE | xxxx | 500 | Number of events to aggregate for one output request the %s in the httpformat will be replaced by a LIST of events.
A value of 1 means no aggregation.
Example: %s = [{'event1': 'data1'},{'event2': 'data2'},...] | +| --httpformattype | HTTP(S) | ULS_HTTP_FORMAT_TYPE | 'JSON-LIST',SINGLE-EVENT' | 'JSON-LIST' | Specifies the type how the given http format is being wrapped (controls, how the httpformat is being rendered in http output) | | | | | | | | | --filehandler | FILE | ULS_FILE_HANDLER | 'SIZE','TIME' | SIZE | Select the handler which decides how the files are rotated if either specific SIZE or TIME has been reached | | --filename | FILE | ULS_FILE_NAME | '/path/to/file.name' | None | The PATH + FILENAME where ULS should create the file | diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cbbf185..a2bcbbb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,25 @@ # Version History +## v1.7.0 +||| +|---|---| +|Date|2023-10-10 +|Kind| FEATURE release +|Author|mschiess@akamai.com, androcho@akamai.com +- **Features** + - Allowing the configuration of the HTTPFORMATTYPE, which controls the building of payloads for aggregated HTTP requests (click [here](FAQ.md#what-is-http-formattype) for additional information) + - Allow adjustment of the "INPUT QUEUE SIZE" threshold (--inputqueuesize) in order to handle huge API pages and fast API output + - New feed for EAA: Directory Health (dirhealth) to fetch health details for configured directories wihtin EAA +- **Minor improvements** + - Added additional checking in the auto installer + - [docker] bumped python version to "3.12.0" + - [docker] bumped GC-LOGS version to "0.0.2(beta)", now supporting credentials in ENV VARS + - [docker] bumped CLI-EAA to "0.6.2" + - [docker] bumped CLI-ETP version to "0.4.4" - fixed a bug in output ordering + empty response handling. + - "get_uls.sh" now allows selection of OS package installation rather than pip3. [See](https://github.com/akamai/uls/issues/46) for more information +- **Housekeeping** + - DocFix Readme.md (thx [@ihommani](https://github.com/akamai/uls/pull/47)) + - Increased default input_queue_size from 10000 to 15000 to avoid race conditions when an API is answering very fast + ## v1.6.6 ||| diff --git a/docs/FAQ.md b/docs/FAQ.md index 148a95a..d4aa14a 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -12,6 +12,8 @@ - [ULS does not start due to missing field in config](#uls-does-not-start-due-to-missing-field-in-config) - [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc) - [WHY JMESPATH and not JSONPATH](#why-jmespath-and-not-jsonpath) +- [What is HTTP FORMATTYPE](#what-is-http-formattype) +- ---- ## FAQ @@ -145,3 +147,30 @@ JMESPATH has a very stable and [well defined language specification](https://jme This gives a user way more options than "pure" jsonpath and is also the reason we decided to go along with the more flexible integration. --- +### What is HTTP FORMATTYPE +As some SIEM operate way more performant, when not having to parse JSON to separate log lines when receiving HTTP requests, ULS 1.6.7 introduces a way to actually control the behavior how data is sent within an HTTP request. +While using the `--httpformattype` flag or the `ULS_HTTP_FORMAT_TYPE` ENV variable the following options can be choosen: +#### JSON-LIST +The HTTP paypload will be a concatenated list of log-lines which will replace the %s variable within the HTTP FORMAT. + +**Example:** +HTTP_FORMAT: `'{"event": %s}'` +Aggregated list: `[{logline1},{logline2},{logline3},{….},{logline500}]` +Final Output Example: `'{"event": [{logline1},{logline2},{logline3},{….},{logline500}]}'` + + +#### SINGLE-EVENT +The HTTP payload will be a single HTTP FORMAT type filled with one logline but still containing multiple loglines per paypload + +**Example:** +HTTP_FORMAT: `'{"event": %s}'` +Aggregated list: `[{logline1},{logline2},{logline3},{….},{logline500}]` +Final Output Example: '{"event": {logline1}}{"event": {logline2}}{"event": {….}}{"event": {logline500}}' + +Within the `single-event` mode, you can freely amend line breake configuration like `\n` or others, by amending it to the HTTP_FORMAT e.g. HTTP_FORMAT: `'{"event": %s}\n'` + +--- +### Error: "Capacity exceeded, too many incoming data vs. slow output" +This error indicates, that more data is coming in to ULS than it can send towards the sepcified output. +As this might be an indication for I/O problems either on the ULS output or the receiving system, it could also just be a specific race condition when the API operations with big pages or at a high speed (e.g. within local LAN). +If requried, the size can be adjusted by using the "--inputqueuesize" introduced in ULS 1.6.7. \ No newline at end of file diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md index c988c72..5f10e64 100644 --- a/docs/LOG_OVERVIEW.md +++ b/docs/LOG_OVERVIEW.md @@ -6,6 +6,7 @@ To get the highest value out of the ingested data, it is crucial to understand t Here are some examples (per product) and links to additional information. ## Table of contents + - [Log Overview](#log-overview) - [Table of contents](#table-of-contents) - [Enterprise Application Access (EAA)](#enterprise-application-access-eaa) @@ -13,10 +14,11 @@ Here are some examples (per product) and links to additional information. - [Admin Logs (ADMIN)](#admin-logs-admin) - [Connector Health (CONHEALTH)](#connector-health-conhealth) - [Device Posture Inventory (DEVINV)](#device-posture-inventory-devinv) - - [Enterprise Threat Protector (ETP)](#enterprise-threat-protector-etp) + - [Directory Health (DIRHEALTH)](#directory-health-dirhealth) + - [Secure Internet Access Enterprise (SIA-E)](#secure-internet-access-enterprise-sia-e) - [Threat Log (THREAT)](#threat-log-threat) - [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup) - - [DNS](#dns) + - [DNS Activity](#dns-activity) - [PROXY](#proxy) - [NETCON](#netcon) - [Akamai MFA (MFA)](#akamai-mfa-mfa) @@ -24,9 +26,8 @@ Here are some examples (per product) and links to additional information. - [Guardicore](#guardicore) - [NETLOG](#netlog) - [INCIDENT](#incident) - - AGENT - - SYSTEM - - Linode + - [Linode](#linode) + - [AUDIT Logs](#audit-logs) ## Enterprise Application Access (EAA) @@ -97,6 +98,7 @@ Additional information regarding the log fields can be found on [here](https://t "privateip": "10.1.4.206", "publicip": "123.123.123.123", "debugchan": "Y", + "datetime": "2021-07-23T18:06:35.676Z", "ts": "2021-07-23T18:06:35.676Z", "cpu": 1.3, "disk": 34.4, @@ -251,10 +253,51 @@ Each event will be one device as a JSON document, example provided with the cli- ``` -## Enterprise Threat Protector (ETP) +### Directory Health (DIRHEALTH) + +Each event will be one directory as a JSON document. +Examples provided can be obtained using cli-eaa command `akamai eaa dir list --json|jq .` +Schema is documented on the [EAA Directory List API doc](https://techdocs.akamai.com/eaa-api/reference/get-directories). + +
+ View directory health event example (JSON) + +```json +{ + "dir_id": "dir://49L59MSsQcyeaRz6N8iKmA", + "service": "ActiveDirectory", + "name": "gc-eaa-forrestor-ActiveDirectory", + "datetime": "2023-10-06T22:02:00.112396+00:00", + "status": 1, + "connector_count": 1, + "directory_status": "ok", + "group_count": 3, + "user_count": 8, + "last_sync": "2023-10-06T15:55:31.026068", + "sync_state": "Dirty", + "conf_state": 1 +} +``` +
+ +## Secure Internet Access Enterprise (SIA-E) + +Formerly known as Enterprise Threat Protector (ETP). + +For large volume of security events (multiple 100K per hour), configure the underlying +`cli-etp` to issue concurrent API requests. + +Depending on your ULS setup you need to pass the `CLIETP_FETCH_CONCURRENT` environment variable. +We recommend to start with the value `2`, observe, and increase up to `8` if you observe backlog. + +This will have a small impact on CPU usage, while increasing the number of events. ### Threat Log (THREAT) Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-threat-event-details) + +
+ Security Threat Event example (JSON) + ```json { "pageInfo": { @@ -643,9 +686,13 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### Accceptable Use Policy Logs (AUP) Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/get-events-details) + +
+ Acceptable Use Policy Event example (JSON) ```json { "pageInfo": { @@ -1034,9 +1081,13 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
-### DNS +### DNS Activity Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-dns-activities-details) + +
+ DNS Activity Event example (JSON) ```json { "pageInfo": { @@ -1261,9 +1312,14 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### PROXY + Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-traffic-transaction-details) + +
+ Proxy Activity Event example (JSON) ```json { "pageInfo": { @@ -2086,12 +2142,54 @@ Additional information regarding the log fields can be found [here](https://tech ] } ``` +
### NETCON Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-network-traffic-connections-details) + +
+ Network Connection Event example (JSON) ```json -{"id": "123", "connectionId": "0xABCDEF1234567890", "domain": "123.123.123.123", "connStartTime": "2023-08-23T07:59:11Z", "connEndTime": "2023-08-23T07:59:11Z", "clientIP": "222.111.222.111", "clientPort": 35593, "destinationIP": "111.222.111.222", "destinationPort": 80, "siteId": 1234536, "siteName": "ETP DEMO", "policyAction": "onramp", "onrampType": "explicit_proxy_tls", "internalClientIP": "", "httpVersion": "N/A", "httpUserAgent": "", "machineId": "", "machineName": "", "clientRequestId": "", "ovfActionId": -1, "ovfActionName": "N/A", "stats": {"httpRequestCount": 1, "inBytes": 0, "outBytes": 0}, "dropInfo": {"wasDropped": true, "droppedReason": "Destination Filter - Internal Host IP"}, "encryptedInternalClientIP": "123123123123123123/ABCDEF", "decryptedInternalClientIP": "192.168.11.168", "sublocationId": "-1", "sublocationName": "N/A", "deviceOwnerId": "", "encryptedInternalClientName": ""} +{ + "id": "123", + "connectionId": "0xABCDEF1234567890", + "domain": "123.123.123.123", + "connStartTime": "2023-08-23T07:59:11Z", + "connEndTime": "2023-08-23T07:59:11Z", + "clientIP": "222.111.222.111", + "clientPort": 35593, + "destinationIP": "111.222.111.222", + "destinationPort": 80, + "siteId": 1234536, + "siteName": "ETP DEMO", + "policyAction": "onramp", + "onrampType": "explicit_proxy_tls", + "internalClientIP": "", + "httpVersion": "N/A", + "httpUserAgent": "", + "machineId": "", + "machineName": "", + "clientRequestId": "", + "ovfActionId": -1, + "ovfActionName": "N/A", + "stats": { + "httpRequestCount": 1, + "inBytes": 0, + "outBytes": 0 + }, + "dropInfo": { + "wasDropped": true, + "droppedReason": "Destination Filter - Internal Host IP" + }, + "encryptedInternalClientIP": "123123123123123123/ABCDEF", + "decryptedInternalClientIP": "192.168.11.168", + "sublocationId": "-1", + "sublocationName": "N/A", + "deviceOwnerId": "", + "encryptedInternalClientName": "" +} ``` +
## Akamai MFA (MFA) Additional information regarding the MFA log fields can be found on [here](https://techdocs.akamai.com/mfa/docs/splunk-app). @@ -2136,44 +2234,42 @@ Authentication Events Example: Guardicore netlog example ```json { -'flow_id':'XXXXX2045a3630852de54dcb99e86f6b06d3969050e153efaed1cb2c4', -'bucket_id':'XXXX-62fa-459a-80e1-c96c2eacdee9', -'source_node_id':'XXXX-3a21-4508-87bd-d0a6512442bc', -'destination_node_id':'UnknownAsset_Internal_192.168.0.0/16', -'source_node_type':'asset', -'destination_node_type':'subnet', -'source_process':'gc-channel', -'destination_process':'Unknown Server (443/TCP)', -'source_process_id':'XXXX491e858806fc17a722c7e93f780e867df4800e6a9bddcc396abf39250', -'destination_process_id':'7XXXX5aad6d81ebe5037013fded72223e12e2d8a0d4e4823c90232139b', -'source_process_name':'gc-channel', -'destination_process_name':'Unknown Server (443/TCP)', -'destination_port':443, -'count':2, -'slot_start_time':XXX72413000, -'incidents':False, -'connection_type':'UNKNOWN', -'source_ip':'192.168.2.76', -'destination_ip':'192.168.2.68', -'ip_protocol':'Tcp', -'source_asset_hash':317458, -'destination_asset_hash':349875, -'violates_policy':False, -'policy_rule':'default', -'policy_ruleset':None, -'policy_verdict':'allowed', -'db_insert_time':'XXXX-11-09T04:09:54.293504', -'id':'XXXXXX-7d27-48b0-ab66-cdedcbc444c3', -'source':{ -'vm':{ -'_id':'XXXXXX-3a21-4508-87bd-d0a6512442bc', -'name':'Gollum Lab Server' -} -}, -'has_mismatch_alert':False, -'original_policy_verdict':'allowed', -'source_process_full_path':'/var/lib/guardicore/sbin/gc-channel', -'destination_process_full_path':None + "id": "123", + "connectionId": "0xABCDEF1234567890", + "domain": "123.123.123.123", + "connStartTime": "2023-08-23T07:59:11Z", + "connEndTime": "2023-08-23T07:59:11Z", + "clientIP": "222.111.222.111", + "clientPort": 35593, + "destinationIP": "111.222.111.222", + "destinationPort": 80, + "siteId": 1234536, + "siteName": "ETP DEMO", + "policyAction": "onramp", + "onrampType": "explicit_proxy_tls", + "internalClientIP": "", + "httpVersion": "N/A", + "httpUserAgent": "", + "machineId": "", + "machineName": "", + "clientRequestId": "", + "ovfActionId": -1, + "ovfActionName": "N/A", + "stats": { + "httpRequestCount": 1, + "inBytes": 0, + "outBytes": 0 + }, + "dropInfo": { + "wasDropped": true, + "droppedReason": "Destination Filter - Internal Host IP" + }, + "encryptedInternalClientIP": "123123123123123123/ABCDEF", + "decryptedInternalClientIP": "192.168.11.168", + "sublocationId": "-1", + "sublocationName": "N/A", + "deviceOwnerId": "", + "encryptedInternalClientName": "" } ``` diff --git a/docs/SIEM/SPLUNK/README.md b/docs/SIEM/SPLUNK/README.md index 555cf79..c40b92c 100644 --- a/docs/SIEM/SPLUNK/README.md +++ b/docs/SIEM/SPLUNK/README.md @@ -3,6 +3,7 @@ ## Table of contents - [Introduction](#introduction) +- [Configuration Options](#configuration-options) - [Additional Splunk Documentation](#additional-splunk-documentation) - [Key Performance Indicators for EAA](#key-performance-indicators-for-eaa) - [[1] No dialout (per connector)](#1-no-dialout-per-connector) @@ -32,12 +33,44 @@ index=akamai source=uls_etp_threat | spath | top event.actionName Splunk also works perfectly with the ULS provided [monitoring data](../../MONITORING.md) +## Configuration Options +### HTTP +Splunk ingestion via HTTP towards the SPLUNK HEC can be done via multiple ways: +- LIST of JSON LOG EVENTS + This method required JSON parsing on the INPUT - minimizing the HTTP payload size. Payload example: + ```json + {"event": [{logevent1},{logevent2},{...}]} + ``` + The required ULS settings look as follows: + ```bash + bin/uls.py \ + -i eaa \ + -f access \ + -o http \ + --httpurl https://splunk-URL:8088/services/collector/event \ + --httpauthheader '{"Authorization": "Splunk 123-321-456"}' + ``` +- SINGLE EVENT + This method allows a raw ingestion of the data, increasing the HTTP payload size. Payload example: + ```json + {"event": {logevent1}}{"event": {logevent2}}{"event": {...}} + ``` + The required ULS settings look as follows: + ```bash + bin/uls.py \ + -i eaa \ + -f access \ + -o http \ + --httpurl https://splunk-URL:8088/services/collector/event \ + --httpauthheader '{"Authorization": "Splunk 123-321-456"}' \ + --httpformattype SINGLE-EVENT + ``` + ## Additional Splunk Documentation - [TCP](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/Monitornetworkports) - [HTTP](https://docs.splunk.com/Documentation/Splunk/8.2.0/Data/UsetheHTTPEventCollector) - [UDP](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/Monitornetworkports) - ## Key Performance Indicators for EAA Splunk search queries provided as examples, feel free to customize them as you see fit. diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index da260b7..0447870 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation type: application version: 2.0.0 -appVersion: "1.6.6" +appVersion: "1.7.0" diff --git a/scripts/get-uls.sh b/scripts/get-uls.sh index cb4fa76..2b8ed58 100755 --- a/scripts/get-uls.sh +++ b/scripts/get-uls.sh @@ -1,6 +1,6 @@ #!/bin/bash # This file will install the latest ULS including all of its modules (latest version) into the current directory/uls -# bash $(curl https://) +# curl -O https://raw.githubusercontent.com/akamai/uls/main/scripts/get-uls.sh && bash get-uls.sh default_modules="eaa,etp,mfa,gc,ln" default_install_dir="$(pwd)/uls" @@ -25,23 +25,44 @@ if [[ -z $(which git) ]] ; then fi ## check python version +### check python exists if [[ -z $(which python3) ]] ; then echo "Python3 binary was not found - exiting" exit 1 fi +### check python version is correct for ULS py_version=$(python3 --version | cut -d " " -f 2) if [[ $(echo ${py_version} | cut -d "." -f 1 ) -ne 3 ]] || [[ $(echo ${py_version} | cut -d "." -f 2 ) -lt 9 ]]; then echo "Wrong Python version - exiting" exit 1 fi + ## pip3 if [[ -z $(which pip3) ]] ; then echo "pip3 binary was not found - exiting" exit 1 fi +### Show versions to verify the correct binaries for python and pip are being used +echo "We will use the following python binaries to install ULS:" +echo -ne "python3: \t $(ls $(which python3))\n" +echo -ne "pip3: \t\t $(ls $(which pip3))\n\n" +echo -ne "Is this correct (Y|n)" +read py_reply + + +case $py_reply in + n|N) + echo -e "Not the right version ?\nPlease adjust your ENV and SYMLINK settings to point to the correct binaries." + echo -e "EXITING !" + exit 1 + ;; + *) + echo "Continuing" +esac + # ASK for INSTALL DETAILS ## Install DIR echo -n "Installation Dir [Default: \"$default_install_dir\"] (ENTER for default or new value): " @@ -83,19 +104,43 @@ if [[ -f "${install_dir}/bin/uls.py" ]] ; then fi fi -## Continue anywaY ? - - -# Installation +## Continue anyway ? +# Installation ## Grab ULS git clone -q https://github.com/akamai/uls.git $install_dir/ pip3 install -q -r ${install_dir}/bin/requirements.txt +function py_reqs() { + to_install=$(pip3 install --dry-run -r $1 | grep -vi "satisfied") + if [[ ! -z $to_install ]] ; then + echo "We are going to install the following python3 requirements:" + echo "----" + echo $to_install + echo "----" + + echo "Do you want to stop here and install OS packages instead [y|N]?" + read package_stop + case $package_stop in + y|Y) + echo "Stopping installation - please install the requried packages manually - exiting" + exit 1 + ;; + *) + echo "installing packages via PIP now" + ;; + esac + fi + + +} + ## Grab EAA-CLI if [[ "$install_modules" == *"eaa"* ]] ; then echo "Installing EAA-CLI" git clone -q --depth 1 --single-branch https://github.com/akamai/cli-eaa.git ${install_dir}/ext/cli-eaa + #echo "${install_dir}/ext/cli-eaa/requirements.txt" + py_reqs ${install_dir}/ext/cli-eaa/requirements.txt pip3 install -q -r ${install_dir}/ext/cli-eaa/requirements.txt fi @@ -103,6 +148,7 @@ fi if [[ "$install_modules" == *"etp"* ]] ; then echo "Installing ETP-CLI" git clone -q --depth 1 --single-branch https://github.com/akamai/cli-etp.git ${install_dir}/ext/cli-etp + py_reqs ${install_dir}/ext/cli-etp/requirements.txt pip3 install -q -r ${install_dir}/ext/cli-etp/requirements.txt fi @@ -110,6 +156,7 @@ fi if [[ "$install_modules" == *"mfa"* ]] ; then echo "Installing MFA-CLI" git clone -q --depth 1 --single-branch https://github.com/akamai/cli-mfa.git ${install_dir}/ext/cli-mfa + py_reqs ${install_dir}/ext/cli-mfa/requirements.txt pip3 install -q -r ${install_dir}/ext/cli-mfa/requirements.txt fi @@ -117,6 +164,7 @@ fi if [[ "$install_modules" == *"gc"* ]] ; then echo "Installing GC-CLI" git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/gc-logs.git ${install_dir}/ext/cli-gc + py_reqs ${install_dir}/ext/cli-gc/bin/requirements.txt pip3 install -q -r ${install_dir}/ext/cli-gc/bin/requirements.txt fi @@ -125,6 +173,7 @@ fi if [[ "$install_modules" == *"ln"* ]] ; then echo "Installing LINODE-CLI" git clone -q --depth 1 -b dev --single-branch https://github.com/MikeSchiessl/ln-logs.git ${install_dir}/ext/cli-linode + py_reqs ${install_dir}/ext/cli-linode/bin/requirements.txt pip3 install -q -r ${install_dir}/ext/cli-linode/bin/requirements.txt fi diff --git a/test/positive_test.bats b/test/positive_test.bats index 751bafc..49a9040 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -11,9 +11,9 @@ mocked_edgerc=FALSE # TIMEOUT # How much time is timeout alklowed to run - uls_test_timeout=20 + uls_test_timeout=90 # Send a kill signal after - uls_kill_timeout=30 + uls_kill_timeout=110 # Used for regular timeout uls_timeout_signal="TERM" uls_timeout_params=" --preserve-status --kill-after $uls_kill_timeout --signal ${uls_timeout_signal} ${uls_test_timeout} " @@ -97,6 +97,16 @@ load 'bats/bats-assert/load.bash' #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } +@test "EAA - DIRHEALTH" { + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed dirhealth --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $eaa_devinv_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout +} + ## ETP @test "ETP - THREAT" { @@ -139,6 +149,16 @@ load 'bats/bats-assert/load.bash' [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } +@test "ETP - NETCON" { + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed netcon --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout +} + ## MFA @test "MFA - EVENT" { run timeout ${uls_timeout_params} ${uls_bin} --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info