add e2e flow to parse events

mtulio · Nov 8, 2024 · 9653870 · 9653870
1 parent 77d61c7
commit 9653870
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 0 deletions.
diff --git a/docs/guides/ocp-perms-track-aws-cloudtrail.md b/docs/guides/ocp-perms-track-aws-cloudtrail.md
@@ -489,3 +489,61 @@ Process CredentialsRequests versus required (API calls):
     --installer-user-name ${INSTALL_USER} \
     --filters cluster-name=lab-trail
 ```
+
+---
+
+## Wait for objects has been created
+
+Watch the objects waiting to collect the events from the time window
+
+```sh
+./cci-events-aws --command collect-events \
+  --bucket-name $BUCKET_NAME --bucket-region us-east-2 \
+  --output events-dev/ \
+  --filters event-region=us-east-2,utc-start-date=2024-09-23-22:00,utc-end-date=2024-09-24-00:00
+```
+
+
+Sync events:
+
+```sh
+mkdir events-openshift-dev
+aws s3 --region us-east-2 sync s3://installer-debug-cloudtrail-logs-*-9d9aefb6/ events-openshift-dev/
+
+# synching to get the final events of cluster finished
+aws s3 --region us-east-2 sync s3://installer-debug-cloudtrail-logs-*-9d9aefb6/AWSLogs/*/CloudTrail/us-east-2/2024/09/23 events-openshift-dev/AWSLogs/*/CloudTrail/us-east-2/2024/09/23
+
+# copying events tarted lower than 20:30 UTC
+mkdir events-openshift-dev-filtered
+cp -rf events-openshift-dev/AWSLogs/*/CloudTrail/us-east-2/2024/09/23/*_CloudTrail_us-east-2_20240923T20* events-openshift-dev-filtered/
+cp -rf events-openshift-dev/AWSLogs/*/CloudTrail/us-east-2/2024/09/23/*_CloudTrail_us-east-2_20240923T21* events-openshift-dev-filtered/
+cp -rf events-openshift-dev/AWSLogs/*/CloudTrail/us-east-2/2024/09/23/*_CloudTrail_us-east-2_20240923T22* events-openshift-dev-filtered/
+
+$ zcat events-openshift-dev-filtered2/* | jq '.Records[].eventTime'| sort -n | head -n1
+"2024-09-23T17:21:21Z"
+$ zcat events-openshift-dev-filtered2/* | jq '.Records[].eventTime'| sort -n | tail -n1
+"2024-09-23T19:53:28Z"
+
+EVENTS_INPUT_RAW=events-openshift-dev-filtered
+EVENTS_OUTPUT_DIR=events-openshift-dev-filtered-output
+INSTALLER_USER=rdo-marco-perms
+CLUSTER_PREFIX=rdo-installer
+mkdir ${EVENTS_OUTPUT_DIR}
+./cci --command extract --events-path ${EVENTS_INPUT_RAW} --output ${EVENTS_OUTPUT_DIR} \
+    --filters principal-name=${INSTALLER_USER}
+
+oc adm release extract \
+    --credentials-requests \
+    --cloud=aws \
+    --to=$PWD/4.18-credrequests \
+    --from=quay.io/openshift-release-dev/ocp-release:4.18.0-ec.1-x86_64
+
+./cci --command compare --events-path ${EVENTS_OUTPUT_DIR}/events.json \
+  --output ${EVENTS_OUTPUT_DIR} \
+  --credentials-requests-path $PWD/4.18-credrequests \
+  --installer-requests-file 4.18-mint-creds.json \
+  --installer-user-name ${INSTALLER_USER} \
+  --filters cluster-name=${CLUSTER_PREFIX}
+
+```
+
diff --git a/labs/ocp-identity/cloud-credentials-insights/cci.py b/labs/ocp-identity/cloud-credentials-insights/cci.py
@@ -72,6 +72,10 @@ def __init__(self, output_dir, filters=None):
         self.processed_files = []
 
     def create_filters(self, filters):
+        """
+        Filters are key values with '=' delimiator, with command sepparated for each filter.
+        Example: filter1=value,filter2=value
+        """
         if filters is None:
             return None
         finalFilters = {}
@@ -356,7 +360,13 @@ def compare(self, opts):
                     self.compiled_users['users'][principal_id]['msg'] = f"no requests file has been found to installer user {principal_id}"
                     self.compiled_users['users'][principal_id]['requested'] = []
                     continue
+
                 reqInstaller = self.credentials_requests.get(principal_id, {})
+                if len(reqInstaller.get('Statement', [])) == 0:
+                    self.compiled_users['users'][principal_id]['msg'] = f"invalid requests file to installer user {principal_id}"
+                    self.compiled_users['users'][principal_id]['requested'] = []
+                    continue
+
                 self.compiled_users['users'][principal_id]['requested'] = reqInstaller.get('Statement', [])[0].get('Action', [])
 
                 # calculate diff