diff --git a/Dockerfile b/Dockerfile index e43605d..6b1652c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bullseye +FROM python:3.10.4-slim-bullseye LABEL MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com" LABEL APP_LONG="Akamai Universal Log Streamer" LABEL APP_SHORT="ULS" @@ -10,8 +10,8 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.3.7" -ARG EAA_CLI_VERSION="0.4.6" +ARG ETP_CLI_VERSION="0.3.8" +ARG EAA_CLI_VERSION="0.5.0" ARG MFA_CLI_VERSION="0.0.9" # ENV VARS diff --git a/README.md b/README.md index a9b9b08..130dc97 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box. ULS can send data into any SIEM that supports either file, TCP, UDP or HTTP ingestion. -It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts. + +It can be run directly as Python code, as a provided Docker container, through `docker compose` scripts or through helm within kubernetes. + ![ULS docker compose usage](docs/images/uls_docker-compose_complex_example.png) @@ -38,6 +40,7 @@ It can be run directly as Python code, as a provided Docker container or through - [ACCESS](docs/LOG_OVERVIEW.md#access-logs-access) - [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin) - [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth) + - [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv) - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) - [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat) - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup) @@ -75,8 +78,9 @@ It can be run directly as Python code, as a provided Docker container or through ## Documentation ULS can be operated in many ways. -Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM. -More information for specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation. +Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM. +ULS is designed as an abstraction layer to easily ship log data to most SIEMs on the market. It does not create any dashboards / data extractions. +Anyway, details for some specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation. ### Generic Requirements - Python 3.9+ OR docker / docker-compose OR kubernetes / helm @@ -92,7 +96,7 @@ Example commands: python3.9 bin/uls.py --input etp --feed threat --output raw # EAA - ACCESS to TCP -python3.0 bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081 +python3.9 bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081 ``` For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md) @@ -111,7 +115,7 @@ For more information, please visit [this document](./docs/KUBERNETES_USAGE.md) ## Development For the latest stable version of this software, please check the [release section](https://github.com/akamai/uls/releases) of the repo. The `main` [branch](https://github.com/akamai/uls) will retain the stable versions. -To ensure a continuous development of this tool, all new updates will go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo. +To ensure a continuous development of this tool, all new updates will first go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo. The `development` branch can be subject to change and could also represent a broken version of this software. In parallel, all new versions within the "main" branch will also be available on the [ULS docker hub space](https://hub.docker.com/repository/docker/akamai/uls). diff --git a/bin/config/global_config.py b/bin/config/global_config.py index 05c1339..ed11679 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.3.5" +__version__ = "1.4.0" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -19,7 +19,7 @@ # Path to the EAA CLI Executable bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa" # Available EAA CLI feeds -eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH'] +eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'] # ETP # Path to the ETP CLI Executable diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 6a6c0e2..446e4a6 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -160,6 +160,16 @@ def proc_create(self): 'list', '--perf', '--tail'] + elif my_feed == "DEVINV": + UlsTools.uls_check_edgerc(self.credentials_file, + self.credentials_file_section, + uls_config.edgerc_openapi) + cli_command = [self.bin_python, + '-u', + product_path, + 'dp', + 'inventory', + '--tail'] else: UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section, diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index 59013cc..3be24a5 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -12,17 +12,17 @@ The following tables list all available command line parameters and their corres ## INPUT -| Parameter | Env - Var | Options | Default | Description | -|---------------------------|-----------------|------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | -| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT' | None | Specify the desired INPUT feed | -| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | -| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | -| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | -| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | -| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | -| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | -| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| Parameter | Env - Var | Options | Default | Description | +|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | +| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT' | None | Specify the desired INPUT feed | +| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | +| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | +| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | +| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | +| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | ## OUTPUT diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9f9c848..cfd6974 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,8 +1,28 @@ # Version History + +## v1.4.0 +||| +|---|---| +|Date|2022-05-XX +|Kind| FEATURE release +|Author|mschiess@akamai.com, adrocho@akamai.com +- **Features** + - Device Inventory (DEVINV) feed added for EAA (requires eaa-cli >= 0.4.9.1) + + +- **Minor improvements** + - bumped EAA to version v0.5.0 + - bumped ETP to version v0.3.8 + - bumped python to version 3.10.4-slim-bullseye + + +- **Bugfix** + - Fixed a bug in the test scripts to support real .edgerc + ## v1.3.5 ||| |---|---| -|Date|2022-04-06 +|Date|2022-04-05 |Kind| Bugfix release |Author|mschiess@akamai.com, adrocho@akamai.com - **Minor improvements** @@ -33,6 +53,7 @@ - Amended new dates to the file headers - Added volume to dockerfile as data storage for "autoresume" + ## v1.3.2 ||| |---|---| diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md index df69f66..da82635 100644 --- a/docs/LOG_OVERVIEW.md +++ b/docs/LOG_OVERVIEW.md @@ -12,6 +12,8 @@ Here are some examples (per product) and links to additional information. - [Access Logs (ACCESS)](#access-logs-access) - [Admin Logs (ADMIN)](#admin-logs-admin) - [Connector Health (CONHEALTH)](#connector-health-conhealth) + - [Device Posture Inventory (DEVINV)](#device-posture-inventory-devinv) + - [Enterprise Threat Protector (ETP)](#enterprise-threat-protector-etp) - [Threat Log (THREAT)](#threat-log-threat) - [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup) @@ -100,6 +102,149 @@ Additional information regarding the log fields can be found on [here](https://t } ``` +### Device Posture Inventory (DEVINV) + +When enabled, EAA can provide a full view on the device running EAA Client. +A report is available in Akamai Control Center and can also be extracted using API. +This feed uses the [Device Posture Inventory](https://techdocs.akamai.com/eaa-api/reference/get-device-posture-inventory) in EAA API. + +Each event will be one device as a JSON document, example provided with the cli-eaa command `akamai eaa dp inventory|head -n1|jq .` + +
+ View device inventory event example (JSON) + +```json +{ + "device_id": "5c98021e78e9c393b07145e388c20ace7733ca88ed63ba0790c09e7ed5c58cf7", + "device_name": "sfo-mpw9c", + "risk_posture_tiers": [ + { + "passed": true, + "name": "Low", + "id": 13, + "tier": true + } + ], + "risk_posture_tags": [ + { + "passed": true, + "name": "Healthy iOS", + "id": 597, + "tier": false + }, + { + "passed": true, + "name": "latest-of-latest", + "id": 949, + "tier": false + }, + { + "passed": true, + "name": "ETP-Healthy-NotCompromised", + "id": 1831, + "tier": false + }, + { + "passed": false, + "remediations": [ + "Unsupported operating system." + ], + "name": "Demo Tag - CB", + "id": 2380, + "tier": false + }, + { + "passed": false, + "remediations": [ + "Unsupported operating system." + ], + "name": "Demo - Tag - Anti malware", + "id": 2381, + "tier": false + }, + { + "passed": true, + "name": "Device - Not Compromised", + "id": 2392, + "tier": false + }, + { + "passed": true, + "name": "Forrester Demo -", + "id": 2402, + "tier": false + }, + { + "passed": true, + "name": "Demo - Anti Malware", + "id": 2407, + "tier": false + }, + { + "passed": true, + "name": "Demo Tag", + "id": 2408, + "tier": false + } + ], + "client_version": "2.7.1", + "idp_username": "N/A", + "user_id": "androcho", + "browsers": [ + { + "name": "Edge", + "version": "101.0.1210.47" + }, + { + "name": "Chrome", + "version": "101.0.4951.64" + }, + { + "name": "Safari", + "version": "15.4" + }, + { + "name": "Firefox", + "version": "100.0" + } +], +"os_name": "macOS", +"os_version": "Monterey 12.3.1 (21E258)", +"signal_timestamp": "2022-05-16T20:21:33.321539+00:00", +"os_update_timestamp": "2022-04-15T20:18:43Z", +"os_auto_update": true, +"anti_malware_running": [ + "Sentinel Agent" +], +"anti_malware_status": [ + { + "name": "Any Vendor", + "passed": true + } +], +"anti_malware_info": [ + { + "name": "Sentinel Agent", + "passed": true + } +], +"firewall_status": "good", +"system_disk_encryption": true, +"etp_client_status": "installed", +"mobile_device": false, +"certificate_profile": [ + { + "name": "cert", + "passed": false + } +], +"etp_signals": { + "threat_detected": false +} +} +``` +
+ ## Enterprise Threat Protector (ETP) ### Threat Log (THREAT) diff --git a/docs/SIEM/QRADAR/README.md b/docs/SIEM/QRADAR/README.md index 483ae13..a3bb897 100644 --- a/docs/SIEM/QRADAR/README.md +++ b/docs/SIEM/QRADAR/README.md @@ -3,7 +3,9 @@ ## Table of contents - [Inputs](#inputs) - - [EAA feeds](#eaa-feeds) + - [EAA Access Log Source Type definition (ACCESS)](#eaa-access-log-source-type-definition-access) + - [EAA Audit logs (ADMIN)](#eaa-audit-logs-admin) + - [EAA Connector Health (CONHEALTH)](#eaa-connector-health-conhealth) - [ULS output configuration](#uls-output-configuration) This document describes how to configure [IBM QRadar](https://www.ibm.com/security/security-intelligence/qradar) in order to receive data from ULS. @@ -14,13 +16,96 @@ Each feed has a corresponding definition file (zip). Use QRadar Extension Manage ## Inputs -### EAA feeds +### EAA Access Log Source Type definition (ACCESS) -The following QRadar DSM version 1.1.2 is able to parse the 3 different EAA feeds coming from a single source host (which is running ULS). +Download the latest QRadar definition for ACCESS feed: [qradar-eaa-access-definition.zip](qradar-eaa-access-definition.zip) -> Download [EAA Combined feeds QRadar DSM version 1.1.2](akamai-eaa-combined-dsm-v1.1.2.zip) +Access fields are mapped as follow: -Last update: March 30th, 2022 +| EAA field | QRadar property | QRadar expression | +| --------- | ----------- | ----------- | +| username | Username | JSON `/"username"` | +| apphost | Application name | JSON `/"apphost"` | +| http_method | - | - | +| url_path | URL | JSON `/"url_path"` | +| http_ver | - | - | +| referer | - | - | +| status_code | Completion code | JSON `/"status_code"` | +| idpinfo | Identity Extended Field | JSON `/"idpinfo"` | +| clientip | Source IP | JSON `/"clientip"` | +| http_verb2 | - | - | +| total_resp_time | - | - | +| connector_resp_time | - | - | +| datetime | Date Time | JSON `/"datetime"` | +| origin_resp_time | - | - | +| origin_host | - | - | +| req_size | - | - | +| content_type | - | - | +| user_agent | - | - | +| device_os | - | - | +| device_type | - | - | +| geo_city | - | - | +| geo_state | - | - | +| geo_statecode | - | - | +| geo_countrycode | - | - | +| geo_country | - | - | +| internal_host | Destination Host Name | Regex `internal_host": "(.*?):(.*?)"` group 1 | +| internal_host | Destination Port | Regex `internal_host": "(.*?):(.*?)"` group 2| +| session_info | - | - | +| groups | Identity Group Name | JSON `/"groups"` | +| session_id | - | - | +| client_id | - | - | +| deny_reason | - | - | +| bytes_out | BytesSent | JSON `/"bytes_out"` | +| bytes_in | BytesReceived | JSON `/"bytes_in"` | +| con_ip | Post NAT Source IP | JSON `/"con_ip"` | +| con_srcport | Post NAT Source Port | JSON `/"con_srcport"` | + +See also [EAA access log definition](https://techdocs.akamai.com/eaa/docs/data-feed-siem#access-logs) for a full definition. + +### EAA Audit logs (ADMIN) + +Download the latest QRadar definition for ADMIN feed: [qradar-eaa-admin-definition.zip](qradar-eaa-admin-definition.zip) + +Admin audit fields are mapped as follow: + +| EAA field | QRadar property | QRadar expression | +| --------- | ----------- | ----------- | +| datetime | Log Source Time | JSON `/"datetime"` | +| username | Username | JSON `/"username"` | +| resource_type | ObjectType | JSON `/"resource_type"` | +| resource | ObjectName | JSON `/"resource"` | +| event | Event ID | JSON `/"event"` | +| event_type | Event Category | JSON `/"event_type"` | + +See also [EAA admin log definition](https://techdocs.akamai.com/eaa/docs/data-feed-siem#admin-logs) for a full definition. + +### EAA Connector Health (CONHEALTH) + +Download the latest QRadar definition for ADMIN feed: [qradar-eaa-conhealth-definition.zip](qradar-eaa-conhealth-definition.zip) + +Connector health fields are mapped as follow: + +| EAA field | QRadar property | QRadar expression | +| --------- | ----------- | ----------- | +| connector_uuid | Resource | JSON `/"connector_uuid"` | +| name | ObjectName | JSON `/"name"` | +| reachable | - | JSON `/"reachable"` | +| status | Completion status | JSON `/"status"` | +| version | Subsystem name | JSON `/"version"` | +| privateip | Source IP | JSON `/"privateip"` | +| publicip | Pre NAT Source IP | JSON `/"publicip"` | +| debugchan | debugchan | JSON `/"debugchan"` | +| ts | - | JSON `/"ts"` with Date format `YYYY-MM-DD"T"HH:mm:ss.sss"Z"` | +| cpu | - | JSON `/"cpu"` | +| disk | - | JSON `/"disk"` | +| mem | - | JSON `/"mem"` | +| network | - | JSON `/"network"` | +| dialout_total | - | JSON `/"dialout_total"` | +| dialout_idle | - | JSON `/"dialout_idle"` | +| dialout_active | - | JSON `/"dialout_active"` | + +See also [EAA connector health feed definition](https://techdocs.akamai.com/eaa/docs/data-feed-siem#connector-health) for a full definition. ## ULS output configuration diff --git a/docs/SIEM/QRADAR/akamai-eaa-combined-dsm-v1.1.2.zip b/docs/SIEM/QRADAR/akamai-eaa-combined-dsm-v1.1.2.zip deleted file mode 100644 index 6585e23..0000000 Binary files a/docs/SIEM/QRADAR/akamai-eaa-combined-dsm-v1.1.2.zip and /dev/null differ diff --git a/docs/SIEM/QRADAR/build-sample-event.sh b/docs/SIEM/QRADAR/build-sample-event.sh index 569a035..2c76e44 100644 --- a/docs/SIEM/QRADAR/build-sample-event.sh +++ b/docs/SIEM/QRADAR/build-sample-event.sh @@ -19,18 +19,21 @@ shuf() { awk 'BEGIN {srand(); OFMT="%.17f"} {print rand(), $0}' "$@" | tmp_dir=$(mktemp -d -t ak-uls-qradar-XXXXXXXXXX) echo "Working in temporary directory $tmp_dir" -echo "Fetching access events..." -akamai eaa log admin --start $START --json --output $tmp_dir/eaa_admin.json -echo "Fetching admin audit events..." -akamai eaa log access --start $START --json --output $tmp_dir/eaa_access.json -echo "Fetching connector health events..." -akamai eaa connector list --perf --json > $tmp_dir/eaa_conhealth.json +function print_usage() { + echo "Usage:" + echo "$0 [eaa|etp]" +} -cat $tmp_dir/eaa_admin.json | shuf | head -n 50 > $tmp_dir/eaa_admin_min.json -cat $tmp_dir/eaa_access.json | shuf | head -n 50 > $tmp_dir/eaa_access_min.json -cat $tmp_dir/eaa_conhealth.json | shuf > $tmp_dir/eaa_conhealth_min.json +case "$1" in + + "eaa") + echo "Fetching access events..." + akamai eaa log admin --start $START --json --output $tmp_dir/eaa_admin.json + echo "Fetching admin audit events..." + akamai eaa log access --start $START --json --output $tmp_dir/eaa_access.json + echo "Fetching connector health events..." + akamai eaa connector list --perf --json > $tmp_dir/eaa_conhealth.json -cat $tmp_dir/eaa_admin_min.json $tmp_dir/eaa_access_min.json $tmp_dir/eaa_conhealth_min.json | shuf > eaa_feeds_combined_sample.json stat eaa_feeds_combined_sample.json echo "File 'eaa_feeds_combined_sample.json' created in the current directory ($(pwd))." @@ -48,7 +51,6 @@ case "$1" in akamai eaa log access --start $START --json --output $tmp_dir/eaa_access.json echo "Fetching connector health events..." akamai eaa connector list --perf --json > $tmp_dir/eaa_conhealth.json - cat $tmp_dir/eaa_admin.json | shuf | head -n 50 > $tmp_dir/eaa_admin_min.json cat $tmp_dir/eaa_access.json | shuf | head -n 50 > $tmp_dir/eaa_access_min.json cat $tmp_dir/eaa_conhealth.json | shuf > $tmp_dir/eaa_conhealth_min.json diff --git a/docs/examples/docker-compose/README.md b/docs/examples/docker-compose/README.md index b17f7ec..b52d03c 100644 --- a/docs/examples/docker-compose/README.md +++ b/docs/examples/docker-compose/README.md @@ -6,9 +6,6 @@ The `.env` files control the ULS via dedicated [ENVIRONMENTAL VARIABLES](../../A The [simple](simple/README.md) directory provides a simple example running ULS via `docker compose` The [complex](complex/README.md) directory provides a more "real world" example combining multiple feeds and different outputs. -The [example](examples/README.md) directory provides different configuration snippets. -<<<<<<< HEAD -The [etp-multi-tenant](etp-tenants/README.md) directory shows how logs from different ETP tenants can be collected. -======= + +The [example](examples/README.md) directory provides different configuration snippets. The [etp-multi-tenant](etp-tenants/README.md) directory shows how logs from different ETP tenants can be collected. ->>>>>>> v.1.3.5 diff --git a/docs/examples/docker-compose/etp-tenants/README.md b/docs/examples/docker-compose/etp-tenants/README.md index aa468bb..95405d9 100644 --- a/docs/examples/docker-compose/etp-tenants/README.md +++ b/docs/examples/docker-compose/etp-tenants/README.md @@ -11,6 +11,11 @@ This directory contains configuration examples (for simple copy & paste usage) t ## docker-compose.yml [This file](docker-compose.yml) contains examples for 2 different ETP tenants collecting the same "threat" feed. +This directory contains configuration examples (for simple copy & paste usage) + +## docker-compose.yml +This file contains examples for 2 different ETP tenants collecting the same "threat" log + ## ENV files The files contains all available ENV VARS explained in a single file. @@ -18,5 +23,4 @@ Tenant 1 [etp-threat-tenant-1.env](./etp-threat-tenant-1.env) Tenant 2 [etp-threat-tenant-2.env](./etp-threat-tenant-2.env) ## EDGERC example - This is a sample `.edgerc` file explaining the ["ETP Multi Tenant support"](./.edgerc-example) \ No newline at end of file diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 4819ca9..2c1f090 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -3,5 +3,5 @@ name: akamai-uls description: Akamai Universal Log Streamer Helm installation type: application -version: 1.3.5 -appVersion: "1.3.5" +version: 1.4.0 +appVersion: "1.4.0" \ No newline at end of file diff --git a/test/README.md b/test/README.md index fdd738e..264feb2 100644 --- a/test/README.md +++ b/test/README.md @@ -8,7 +8,8 @@ bash test/test.sh For better testing stability, we packed the required dependencies into the test dir as well. ### Requirements -- [bats](https://bats-core.readthedocs.io/en/stable/) +- [bats](https://bats-core.readthedocs.io/en/stable/) `bats --version Bats 1.6.0` - timeout command is available - uls and the cli's are installed -- working (fully fledged .edgerc file - not necessarily providing logs) \ No newline at end of file +- working (fully fledged .edgerc file - or the inline MOCKED one) + - mocked edgerc is currently failing with EAA \ No newline at end of file diff --git a/test/negative_test.bats b/test/negative_test.bats index 8609c5c..8b7fcc8 100644 --- a/test/negative_test.bats +++ b/test/negative_test.bats @@ -26,7 +26,7 @@ load 'bats/bats-assert/load.bash' } @test "Input / Feed mismatch" { run $uls_bin --input eaa --feed auth -o raw - assert_output --partial "Feed (AUTH) not available - Available: ['ACCESS', 'ADMIN', 'CONHEALTH']" + assert_output --partial "Feed (AUTH) not available - Available:" [ "$status" -eq 1 ] } @test "Checking --feed and --intput etp" { diff --git a/test/positive_test.bats b/test/positive_test.bats index 7b8b450..39589fd 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -4,19 +4,36 @@ ## THIS Should be run from ULS ROOT DIR # Variables - # ULS Binary + # ULS Binary uls_bin=bin/uls.py - # TESTING EDGERC FILE -#uls_edgerc=~/.edgerc -uls_edgerc=test/_mocked_edgerc - - # TESTING SECTION -uls_section=testing - - # TIMEOUT -uls_test_timeout=30 - + # Should we using a mocked edgerc (TRUE/FALSE) ? +mocked_edgerc=FALSE + + # TIMEOUT +uls_test_timeout=60 + + +### Switch between mocked and real edgerc +if [ "$mocked_edgerc"=="FALSE" ] ; then + # REAL EDGERC FILE + uls_edgerc=~/.edgerc + uls_section=akamaidemo + # Variables + eaa_access_assert="username" + eaa_devinv_assert="client_version" + etp_assert="configId" + jmespath_assert="['" +else + # TESTING EDGERC FILE & section + uls_edgerc=test/_mocked_edgerc + uls_section=testing + # Variables + eaa_access_assert="" + eaa_devinv_assert="" + etp_assert="" + jmespath_assert="" +fi # Load support libs load 'bats/bats-support/load.bash' @@ -27,14 +44,14 @@ load 'bats/bats-assert/load.bash' ## EAA @test "EAA - ACCESS" { run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $eaa_access_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - ADMIN" { run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -46,11 +63,18 @@ load 'bats/bats-assert/load.bash' #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout } +@test "EAA - DEVINV" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed devinv --output raw --edgerc $uls_edgerc --section $uls_section + assert_output --partial $eaa_devinv_assert + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} ## ETP @test "ETP - THREAT" { run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -58,7 +82,7 @@ load 'bats/bats-assert/load.bash' @test "ETP - AUP" { run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -66,7 +90,7 @@ load 'bats/bats-assert/load.bash' @test "ETP - DNS" { run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -74,7 +98,7 @@ load 'bats/bats-assert/load.bash' @test "ETP - PROXY" { run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $etp_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -111,7 +135,7 @@ load 'bats/bats-assert/load.bash' ## Transformation @test "TRANSFORM - MCAS" { run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial "detection_time" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -119,7 +143,7 @@ load 'bats/bats-assert/load.bash' @test "TRANSFORM - JMESPATH" { run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section - assert_output "" + assert_output --partial $jmespath_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout @@ -129,10 +153,16 @@ load 'bats/bats-assert/load.bash' @test "AUTORESUME - Create File" { rm -f /tmp/uls_eaa_access.ckpt run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ - assert_output "" + assert_output --partial $eaa_access_assert #assert_output --partial " seems to be empty" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout rm -f /tmp/uls_eaa_access.ckpt +} +## EAA +@test "LINT the HELM CHART" { + run helm lint docs/examples/kubernetes/helm/akamai-uls --strict + assert_output --partial "0 chart(s) failed" + [ "$status" -eq 0 ] #return value for Chart Lint: 0 } \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index f8c3a64..7da9818 100644 --- a/test/test.sh +++ b/test/test.sh @@ -43,11 +43,11 @@ if [ "$select" == "all" ] ; then for i in $tests_available ; do echo "> $i" do_test $i + do_tag done else do_test $1 fi -do_tag exit 0