v1.4.0 (#26)

v.1.4.0
akamai · May 20, 2022 · c968695 · c968695
1 parent 82c81f5
commit c968695
Show file tree

Hide file tree

Showing 17 changed files with 369 additions and 70 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM            python:3.10-slim-bullseye
+FROM            python:3.10.4-slim-bullseye
 LABEL 	        MAINTAINER="Mike Schiessl - [email protected]"
 LABEL	        APP_LONG="Akamai Universal Log Streamer"
 LABEL           APP_SHORT="ULS"
@@ -10,8 +10,8 @@ ARG             HOMEDIR="/opt/akamai-uls"
 ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
-ARG             ETP_CLI_VERSION="0.3.7"
-ARG             EAA_CLI_VERSION="0.4.6"
+ARG             ETP_CLI_VERSION="0.3.8"
+ARG             EAA_CLI_VERSION="0.5.0"
 ARG             MFA_CLI_VERSION="0.0.9"
 
 # ENV VARS

diff --git a/README.md b/README.md
@@ -9,7 +9,9 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka
 
 Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box.  
 ULS can send data into any SIEM that supports either file, TCP, UDP or HTTP ingestion.  
-It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts.
+
+It can be run directly as Python code, as a provided Docker container,  through `docker compose` scripts or through helm within kubernetes.
+
 
 
 ![ULS docker compose usage](docs/images/uls_docker-compose_complex_example.png)
@@ -38,6 +40,7 @@ It can be run directly as Python code, as a provided Docker container or through
       - [ACCESS](docs/LOG_OVERVIEW.md#access-logs-access) 
       - [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin)
       - [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth)
+      - [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv)
     - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
       - [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat)
       - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup)
@@ -75,8 +78,9 @@ It can be run directly as Python code, as a provided Docker container or through
 
 ## Documentation
 ULS can be operated in many ways.  
-Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM.
-More information for specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation.
+Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM.  
+ULS is designed as an abstraction layer to easily ship log data to most SIEMs on the market. It does not create any dashboards / data extractions.  
+Anyway, details for some specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation.
 
 ### Generic Requirements
 - Python 3.9+ OR docker / docker-compose OR kubernetes / helm
@@ -92,7 +96,7 @@ Example commands:
 python3.9 bin/uls.py --input etp --feed threat --output raw
 
 # EAA - ACCESS to TCP
-python3.0  bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081
+python3.9  bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081
 ```
 For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md)
 
@@ -111,7 +115,7 @@ For more information, please visit [this document](./docs/KUBERNETES_USAGE.md)
 ## Development
 
 For the latest stable version of this software, please check the [release section](https://github.com/akamai/uls/releases) of the repo. The `main` [branch](https://github.com/akamai/uls) will retain the stable versions.
-To ensure a continuous development of this tool, all new updates will go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo.  
+To ensure a continuous development of this tool, all new updates will first go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo.  
 The `development` branch can be subject to change and could also represent a broken version of this software.
 In parallel, all new versions within the "main" branch will also be available on the [ULS docker hub space](https://hub.docker.com/repository/docker/akamai/uls).
 

diff --git a/bin/config/global_config.py b/bin/config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.3.5"
+__version__ = "1.4.0"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -19,7 +19,7 @@
         # Path to the EAA CLI Executable
 bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"
         # Available EAA CLI feeds
-eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH']
+eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV']
 
     # ETP
         # Path to the ETP CLI Executable

diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
@@ -160,6 +160,16 @@ def proc_create(self):
                                        'list',
                                        '--perf',
                                        '--tail']
+                    elif my_feed == "DEVINV":
+                        UlsTools.uls_check_edgerc(self.credentials_file,
+                                                  self.credentials_file_section,
+                                                  uls_config.edgerc_openapi)
+                        cli_command = [self.bin_python,
+                                       '-u',
+                                       product_path,
+                                       'dp',
+                                       'inventory',
+                                       '--tail']
                     else:
                         UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,

diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
@@ -12,17 +12,17 @@ The following tables list all available command line parameters and their corres
 
 
 ## INPUT
-| Parameter                 | Env - Var       | Options                                                                                        | Default       | Description                                                                                                                                                                                 |
-|---------------------------|-----------------|------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| -i <br> --input           | ULS_INPUT       | 'EAA', 'ETP', 'MFA'                                                                            | None          | Specify the desired INPUT source                                                                                                                                                            |
-| --feed                    | ULS_FEED        | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'<br> ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'<br> MFA: 'EVENT'  | None          | Specify the desired INPUT feed                                                                                                                                                              |
-| --format                  | ULS_FORMAT      | 'JSON', 'TEXT'                                                                                 | JSON          | Specify the desired INPUT (=OUTPUT) format                                                                                                                                                  |
-| --inproxy<br>--inputproxy | ULS_INPUT_PROXY | HOST:PORT                                                                                      | None          | Adjust proxy usage for INPUT data collection (cli) <br>If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
-| --rawcmd                  | ULS_RAWCMD      | \<cli command\>                                                                                | None          | USE with caution /!\ <br> This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature)                               |
-| --edgerc                  | ULS_EDGERC      | /path/to/your/.edgerc                                                                          | '~/.edgerc'   | Specify the location of the .edgerc EDGE GRID AUTH file                                                                                                                                     |
-| --section                 | ULS_SECTION     | edgerc_config_section                                                                          | 'default'     | Specify the desired section within the .edgerc file                                                                                                                                         |
-| --starttime               | ULS_STARTTIME   | EPOCH timestamp                                                                                | `cli_default` | Specify an EPOCH timestamp from where to start the log collection.                                                                                                                          |
-| --endtime                 | ULS_ENDTIME     | EPOCH timestamp                                                                                | None          | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.<br>ULS will not continue reading logs on CLI errors !!!                                   |
+| Parameter                 | Env - Var       | Options                                                                                                 | Default       | Description                                                                                                                                                                                 |
+|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| -i <br> --input           | ULS_INPUT       | 'EAA', 'ETP', 'MFA'                                                                                     | None          | Specify the desired INPUT source                                                                                                                                                            |
+| --feed                    | ULS_FEED        | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'<br> ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'<br> MFA: 'EVENT' | None          | Specify the desired INPUT feed                                                                                                                                                              |
+| --format                  | ULS_FORMAT      | 'JSON', 'TEXT'                                                                                          | JSON          | Specify the desired INPUT (=OUTPUT) format                                                                                                                                                  |
+| --inproxy<br>--inputproxy | ULS_INPUT_PROXY | HOST:PORT                                                                                               | None          | Adjust proxy usage for INPUT data collection (cli) <br>If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
+| --rawcmd                  | ULS_RAWCMD      | \<cli command\>                                                                                         | None          | USE with caution /!\ <br> This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature)                               |
+| --edgerc                  | ULS_EDGERC      | /path/to/your/.edgerc                                                                                   | '~/.edgerc'   | Specify the location of the .edgerc EDGE GRID AUTH file                                                                                                                                     |
+| --section                 | ULS_SECTION     | edgerc_config_section                                                                                   | 'default'     | Specify the desired section within the .edgerc file                                                                                                                                         |
+| --starttime               | ULS_STARTTIME   | EPOCH timestamp                                                                                         | `cli_default` | Specify an EPOCH timestamp from where to start the log collection.                                                                                                                          |
+| --endtime                 | ULS_ENDTIME     | EPOCH timestamp                                                                                         | None          | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.<br>ULS will not continue reading logs on CLI errors !!!                                   |
 
 
 ## OUTPUT

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,8 +1,28 @@
 # Version History
+
+## v1.4.0
+|||
+|---|---|
+|Date|2022-05-XX
+|Kind| FEATURE release
+|Author|[email protected], [email protected]
+- **Features**
+  - Device Inventory (DEVINV) feed added for EAA (requires eaa-cli >= 0.4.9.1)
+
+
+- **Minor improvements**
+  - bumped EAA to version v0.5.0
+  - bumped ETP to version v0.3.8
+  - bumped python to version 3.10.4-slim-bullseye
+
+
+- **Bugfix**
+  - Fixed a bug in the test scripts to support real .edgerc  
+
 ## v1.3.5
 |||
 |---|---|
-|Date|2022-04-06
+|Date|2022-04-05
 |Kind| Bugfix release
 |Author|[email protected], [email protected]
 - **Minor improvements**
@@ -33,6 +53,7 @@
   - Amended new dates to the file headers
   - Added volume to dockerfile as data storage for "autoresume"
 
+
 ## v1.3.2
 |||
 |---|---|

diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md
@@ -12,6 +12,8 @@ Here are some examples (per product) and links to additional information.
     - [Access Logs (ACCESS)](#access-logs-access)
     - [Admin Logs (ADMIN)](#admin-logs-admin)
     - [Connector Health (CONHEALTH)](#connector-health-conhealth)
+    - [Device Posture Inventory (DEVINV)](#device-posture-inventory-devinv)
+
   - [Enterprise Threat Protector (ETP)](#enterprise-threat-protector-etp)
     - [Threat Log (THREAT)](#threat-log-threat)
     - [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup)
@@ -100,6 +102,149 @@ Additional information regarding the log fields can be found on [here](https://t
 }
 ```
 
+### Device Posture Inventory (DEVINV)
+
+When enabled, EAA can provide a full view on the device running EAA Client.
+A report is available in Akamai Control Center and can also be extracted using API.
+This feed uses the [Device Posture Inventory](https://techdocs.akamai.com/eaa-api/reference/get-device-posture-inventory) in EAA API.
+
+Each event will be one device as a JSON document, example provided with the cli-eaa command `akamai eaa dp inventory|head -n1|jq .`
+
+<details>
+    <summary>View device inventory event example (JSON)</summary>
+
+```json
+{
+    "device_id": "5c98021e78e9c393b07145e388c20ace7733ca88ed63ba0790c09e7ed5c58cf7",
+    "device_name": "sfo-mpw9c",
+    "risk_posture_tiers": [
+        {
+        "passed": true,
+        "name": "Low",
+        "id": 13,
+        "tier": true
+        }
+    ],
+    "risk_posture_tags": [
+        {
+        "passed": true,
+        "name": "Healthy iOS",
+        "id": 597,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "latest-of-latest",
+        "id": 949,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "ETP-Healthy-NotCompromised",
+        "id": 1831,
+        "tier": false
+        },
+        {
+        "passed": false,
+        "remediations": [
+            "Unsupported operating system."
+        ],
+        "name": "Demo Tag - CB",
+        "id": 2380,
+        "tier": false
+        },
+        {
+        "passed": false,
+        "remediations": [
+            "Unsupported operating system."
+        ],
+        "name": "Demo - Tag - Anti malware",
+        "id": 2381,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "Device - Not Compromised",
+        "id": 2392,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "Forrester Demo -",
+        "id": 2402,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "Demo - Anti Malware",
+        "id": 2407,
+        "tier": false
+        },
+        {
+        "passed": true,
+        "name": "Demo Tag",
+        "id": 2408,
+        "tier": false
+        }
+    ],
+    "client_version": "2.7.1",
+    "idp_username": "N/A",
+    "user_id": "androcho",
+    "browsers": [
+        {
+        "name": "Edge",
+        "version": "101.0.1210.47"
+        },
+        {
+        "name": "Chrome",
+        "version": "101.0.4951.64"
+        },
+        {
+        "name": "Safari",
+        "version": "15.4"
+        },
+        {
+        "name": "Firefox",
+        "version": "100.0"
+        }
+],
+"os_name": "macOS",
+"os_version": "Monterey 12.3.1 (21E258)",
+"signal_timestamp": "2022-05-16T20:21:33.321539+00:00",
+"os_update_timestamp": "2022-04-15T20:18:43Z",
+"os_auto_update": true,
+"anti_malware_running": [
+    "Sentinel Agent"
+],
+"anti_malware_status": [
+    {
+    "name": "Any Vendor",
+    "passed": true
+    }
+],
+"anti_malware_info": [
+    {
+    "name": "Sentinel Agent",
+    "passed": true
+    }
+],
+"firewall_status": "good",
+"system_disk_encryption": true,
+"etp_client_status": "installed",
+"mobile_device": false,
+"certificate_profile": [
+    {
+    "name": "cert",
+    "passed": false
+    }
+],
+"etp_signals": {
+    "threat_detected": false
+}
+}
+```
+</details>
+
 ## Enterprise Threat Protector (ETP)
 
 ### Threat Log (THREAT)