From 220a078338a07f8c20d8d02dcd256e8d0087aeed Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 16 Sep 2024 15:56:22 +0000 Subject: [PATCH 01/47] access edge creator first draft --- .vscode/settings.json | 11 +- fixlib/fixlib/baseresources.py | 1 + plugins/aws/fix_plugin_aws/access_edges.py | 678 ++++++++++++++++++++ plugins/aws/fix_plugin_aws/collector.py | 7 +- plugins/aws/fix_plugin_aws/resource/base.py | 23 +- plugins/aws/fix_plugin_aws/resource/s3.py | 8 +- 6 files changed, 720 insertions(+), 8 deletions(-) create mode 100644 plugins/aws/fix_plugin_aws/access_edges.py diff --git a/.vscode/settings.json b/.vscode/settings.json index a86820abbf..5cbd3d3f0f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,7 +11,16 @@ ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": false, - "python.formatting.provider": "black", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + }, + "black-formatter.args": [ + "--line-length", + "120", + "--target-version", + "py39" + ], "search.exclude": { "**/*.code-search": true, "**/bower_components": true, diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 45c68d3e8e..06ad46de04 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -68,6 +68,7 @@ class ModelReference(TypedDict, total=False): class EdgeType(Enum): default = "default" delete = "delete" + access = "access" @staticmethod def from_value(value: Optional[str] = None) -> EdgeType: diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py new file mode 100644 index 0000000000..b71afd0e01 --- /dev/null +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -0,0 +1,678 @@ +from abc import ABC +from enum import Enum, StrEnum +from functools import lru_cache +from attr import define, evolve, frozen +from fix_plugin_aws.resource.base import AwsResource, GraphBuilder + +from typing import Dict, Any, List, Literal, Set, Optional, Tuple, TypedDict, Union, cast, Pattern + +from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser +from fix_plugin_aws.resource.s3 import AwsS3Bucket +from fixcore.web import permission +from fixlib.baseresources import EdgeType +from fixlib.types import Json + +from cloudsplaining.scan.policy_document import PolicyDocument +from cloudsplaining.scan.statement_detail import StatementDetail +from policy_sentry.querying.actions import get_action_data, get_actions_matching_arn +from policy_sentry.analysis.expand import determine_actions_to_expand +from policy_sentry.querying.all import get_all_actions +from policy_sentry.util.arns import ARN +import re +import logging + + +log = logging.getLogger(__name__) + +ALL_ACTIONS = get_all_actions() + +ResourceContstaint = str + + +class PolicySourceKind(StrEnum): + Principal = "principal" # e.g. IAM user, attached policy + Group = "group" # policy comes from an IAM group + Resource = "resource" # e.g. s3 bucket policy + + +class PolicySource(TypedDict): + kind: PolicySourceKind + arn: str + + +class HasResourcePolicy(ABC): + # returns a list of all policies that affects the resource (inline, attached, etc.) + def resource_policy(self, builder: GraphBuilder) -> List[Tuple[PolicySource, Json]]: + raise NotImplementedError + + + +@frozen +class PermissionScope: + source: PolicySource + restriction: str + conditions: List[Json] + + +@frozen +class AccessPermission: + action: str + level: str + scopes: List[PermissionScope] + + +@define +class IamRequestContext: + principal: AwsResource + identity_policies: List[Tuple[PolicySource, PolicyDocument]] + permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too + service_control_policies: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too + # technically we should also add a list of session policies here, but they don't exist in the collector context + + def all_policies( + self, resource_based_policies: Optional[List[Tuple[PolicySource, PolicyDocument]]] = None + ) -> List[Tuple[PolicySource, PolicyDocument]]: + return ( + self.identity_policies + + self.permission_boundaries + + self.service_control_policies + + (resource_based_policies or []) + ) + + +def find_policy_doc(policy: AwsIamPolicy) -> Optional[Json]: + if not policy.policy_document: + return None + + return policy.policy_document.document + + +IamAction = str + + +@lru_cache(maxsize=1024) +def find_allowed_action(policy_document: PolicyDocument) -> Set[IamAction]: + allowed_actions: Set[IamAction] = set() + for statement in policy_document.statements: + statement = cast(StatementDetail, statement) + if statement.effect_allow: + allowed_actions.update(get_expanded_action(statement)) + + return allowed_actions + + +def find_all_allowed_actions(all_involved_policies: List[PolicyDocument]) -> Set[IamAction]: + allowed_actions: Set[IamAction] = set() + for p in all_involved_policies: + allowed_actions.update(find_allowed_action(p)) + return allowed_actions + + +@lru_cache(maxsize=1024) +def get_expanded_action(statement: StatementDetail) -> Set[str]: + actions = set() + expanded: List[str] = statement.expanded_actions or [] # type: ignore + for action in expanded: + actions.add(action) + + return actions + + +@lru_cache(maxsize=1024) +def make_resoruce_regex(aws_resorce_wildcard: str) -> Pattern[str]: + # step 1: translate aws wildcard to python regex + python_regex = aws_resorce_wildcard.replace("*", ".*").replace("?", ".") + # step 2: compile the regex + return re.compile(f"^{python_regex}$", re.IGNORECASE) + +def expand_wildcards_and_match(*, identifier: str, wildcard_string: str) -> bool: + """ + helper function to expand wildcards and match the identifier + + use case: + match the resource constraint (wildcard) with the ARN + match the wildcard action with the specific action + """ + pattern = make_resoruce_regex(wildcard_string) + return pattern.match(identifier) is not None + + +@frozen +class Allowed: + resource_constraint: str + condition: Optional[Json] = None + +@frozen +class Denied: + pass + + + +AccessResult = Union[Allowed, Denied] + + + +def policy_matching_statement_exists(policy: PolicyDocument, effect: Literal["Allow", "Deny"], action: str, resource: AwsResource) -> Optional[Tuple[StatementDetail, Optional[ResourceContstaint]]]: + """ + check if a matching statement exists in the policy, returns boolean if exists or json in case there is a condition we can't resolve + """ + if resource.arn is None: + raise ValueError("Resource ARN is missing, go and fix the filtering logic") + + for statement in policy.statements: + statement = cast(StatementDetail, statement) + + # step 1: check if the effect matches + if statement.effect != effect: + # wrong effect, skip this statement + continue + + # step 2: check if the action matches + action_match = False + if statement.actions: + for a in statement.actions: + if expand_wildcards_and_match(identifier=action, wildcard_string=a): + action_match = True + break + else: + # if not_action is also missing, consider all actions allowed + action_match = True + for na in statement.not_action: + if expand_wildcards_and_match(identifier=action, wildcard_string=na): + action_match = False + break + if not action_match: + # action does not match, skip this statement + continue + + # step 3: check if the resource matches + matched_resource_constraint: Optional[ResourceContstaint] = None + resource_matches = False + if len(statement.resources) > 0: + for resource_constraint in statement.resources: + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): + matched_resource_constraint = resource_constraint + resource_matches = True + break + elif len(statement.not_resource) > 0: + resource_matches = True + for not_resource_constraint in statement.not_resource: + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): + resource_matches = False + break + matched_resource_constraint = "not " + not_resource_constraint + else: + # no Resource/NotResource specified, consider allowed + resource_matches = True + if not resource_matches: + # resource does not match, skip this statement + continue + + # step 4: check if there is a condition to care about + return (statement, matched_resource_constraint) + + return None + + +def check_principal_match(principal: AwsResource, aws_principal_list: List[str]) -> bool: + for aws_principal in aws_principal_list: + if aws_principal == "*": + return True + + if principal.arn == aws_principal: + return True + + if principal.id == aws_principal: + return True + + principal_arn = ARN + if principal_arn.account == aws_principal: # type: ignore + return True + + return False + + +def collect_matching_resource_statements(principal: AwsResource, resoruce_policy: PolicyDocument, action: str, resource: AwsResource) -> List[Tuple[StatementDetail, Optional[ResourceContstaint]]]: + """ + resoruce based policies contain principal field and need to be handled differently + """ + results: List[Tuple[StatementDetail, Optional[ResourceContstaint]]] = [] + + if resource.arn is None: + raise ValueError("Resource ARN is missing, go and fix the filtering logic") + + for statement in resoruce_policy.statements: + statement = cast(StatementDetail, statement) + + # step 1: check the principal + principal_match = False + if policy_principal := statement.json.get("Principal", None): + if policy_principal == "*": + principal_match = True + elif "AWS" in policy_principal: + aws_principal_list = policy_principal["AWS"] + assert isinstance(aws_principal_list, list) + if check_principal_match(principal, aws_principal_list): + principal_match = True + else: + # aws service principal is specified, we do not handle such cases yet + pass + elif policy_not_principal := statement.json.get("NotPrincipal", None): + # * is not allowed in NotPrincipal, so we can skip the check + principal_match = True + if "AWS" in policy_not_principal: + aws_principal_list = policy_not_principal["AWS"] + assert isinstance(aws_principal_list, list) + if check_principal_match(principal, aws_principal_list): + principal_match = False + else: + # aws service principal is specified, we do not handle such cases yet + pass + else: + principal_match = True + + if not principal_match: + # principal does not match, we can shortcut here + continue + + + # step 2: check the action + action_match = False + if statement.actions: + for a in statement.actions: + if expand_wildcards_and_match(identifier=action, wildcard_string=a): + action_match = True + break + else: + # if not_action is also missing, consider all actions allowed + action_match = True + for na in statement.not_action: + if expand_wildcards_and_match(identifier=action, wildcard_string=na): + action_match = False + break + if not action_match: + # action does not match, skip this statement + continue + + # step 3: check if the resource matches + matched_resource_constraint: Optional[ResourceContstaint] = None + resource_matches = False + if len(statement.resources) > 0: + for resource_constraint in statement.resources: + + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): + resource_matches = True + matched_resource_constraint = resource_constraint + break + elif len(statement.not_resource) > 0: + resource_matches = True + for not_resource_constraint in statement.not_resource: + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): + resource_matches = False + break + matched_resource_constraint = "not " + not_resource_constraint + else: + # no Resource/NotResource specified, consider allowed + resource_matches = True + if not resource_matches: + # resource does not match, skip this statement + continue + + results.append((statement, matched_resource_constraint)) + + return results + + + + + +def check_explicit_deny(request_context: IamRequestContext, resource: AwsResource, action: str, resource_based_policies: List[Tuple[PolicySource, PolicyDocument]]) -> Union[Literal['Denied', 'Allowed'], List[Json]]: + + denied_when_any: List[Json] = [] + + # check all the policies except the resource based ones + for source, policy in request_context.all_policies(): + match policy_matching_statement_exists(policy, "Deny", action, resource): + case True: + return 'Denied' + case False: + pass + case condition: + denied_when_any.append(condition) + + for source, policy in resource_based_policies: + # resource based policies require a different handling + resource_policy_statements = collect_matching_resource_statements(request_context.principal, policy, action, resource) + for statement, _ in resource_policy_statements: + if not statement.effect_deny: + continue + if statement.condition: + denied_when_any.append(statement.condition) + else: + return 'Denied' + + if len(denied_when_any) > 0: + return denied_when_any + + return 'Allowed' + + +def scp_allowed(request_context: IamRequestContext) -> bool: + # todo: collect the necessary resources and implement this + return True + + +class ResourcePolicyCheckResult(Enum): + NO_MATCH = 0 + DENY_MATCH = 1 + ALLOW_MATCH = 2 + + +@frozen +class FinalAllow: + scopes: List[PermissionScope] + + +@frozen +class Continue: + scopes: List[PermissionScope] + + +ResourceBasedPolicyResult = Union[FinalAllow, Continue] + + +# check if the resource based policies allow the action +# as a shortcut we return the first allow statement we find, or a first seen condition. +# todo: collect all allow statements and conditions +def check_resource_based_policies( + principal: AwsResource, + action: str, + resource: AwsResource, + resource_based_policies: List[Tuple[PolicySource, PolicyDocument]]) -> ResourceBasedPolicyResult: + + scopes: List[PermissionScope] = [] + + # todo: add special handling for IAM and KMS resources + + for source, policy in resource_based_policies: + matching_statements = collect_matching_resource_statements(principal, policy, action, resource) + if len(matching_statements) == 0: + continue + + for statement, constraint in matching_statements: + if statement.effect_allow: + if statement.condition: + scopes.append(PermissionScope( + source=source, + restriction=constraint or "*", + conditions=[statement.condition] + )) + else: + scopes.append(PermissionScope( + source=source, + restriction=constraint or "*", + conditions=[] + )) + + match principal.kind: + case "aws_iam_user": + # in case of IAM user, access is allowed regardless of the rest of the policies + return FinalAllow(scopes) + + # todo: deal with session principals somehow + case _: + # in case of other IAM principals, allow on resource based policy is not enough and + # we need to check the permission boundaries + return Continue(scopes) + + +def check_identity_based_policies(request_context: IamRequestContext, resource: AwsResource, action: str) -> List[PermissionScope]: + + scopes: List[PermissionScope] = [] + + for source, policy in request_context.identity_policies: + if exists := policy_matching_statement_exists(policy, "Allow", action, resource): + statement, resource_constraint = exists + restriction: str = resource_constraint or "*" + scopes.append(PermissionScope(source, restriction, [statement.condition])) + + return scopes + + +def check_permission_boundaries(request_context: IamRequestContext, resource: AwsResource, action: str) -> Union[bool, List[Json]]: + + conditions: List[Json] = [] + + for source, policy in request_context.permission_boundaries: + if exists := policy_matching_statement_exists(policy, "Allow", action, resource): + statement, constraint = exists + if not statement.condition: + return True + conditions.append(statement.condition) + + if len(conditions) > 0: + return conditions + + # no matching permission boundaries that allow access + return False + + + +def negate_condition(condition: Json) -> Json: + # todo: implement this + return condition + +def merge_conditions(conditions: List[Json]) -> Json: + # todo: implement this + return conditions[0] + + +def get_action_level(action: str) -> str: + service, action_name = action.split(":") + action_data = get_action_data(service, action_name) + level = [info["access_level"] for info in action_data[service] if action == info["action"]][0] + return level + +# logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html#policy-eval-denyallow +def check_policies( + request_context: IamRequestContext, + resource: AwsResource, + action: str, + resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], +) -> Optional[AccessPermission]: + + deny_conditions: Optional[Json] = None + # 1. check for explicit deny. If denied, we can abort immediately + match check_explicit_deny(request_context, resource, action, resource_based_policies): + case 'Denied': + return None + case 'Allowed': + pass + case conditions: + negated = [negate_condition(c) for c in conditions] + merged = merge_conditions(negated) + deny_conditions = merged + + # 2. check for organization SCPs + if len(request_context.service_control_policies) > 0: + org_scp_allowed = scp_allowed(request_context) + if not org_scp_allowed: + return None + + # 3. check resource based policies + resource_based_allowed_scopes: List[PermissionScope] = [] + if len(resource_based_policies) > 0: + match check_resource_based_policies(request_context.principal, action, resource, resource_based_policies): + case FinalAllow(scopes): + allowed_scopes: List[PermissionScope] = [] + for scope in scopes: + if deny_conditions: + new_conditions = [merge_conditions([deny_conditions, c]) for c in scope.conditions] + scope = evolve(scope, conditions=new_conditions) + allowed_scopes.append(scope) + + return AccessPermission( + action=action, + level=get_action_level(action), + scopes=allowed_scopes + ) + + case Continue(scopes): + resource_based_allowed_scopes.extend(scopes) + + # 4. check identity based policies + identity_based_scopes: List[PermissionScope] = [] + if len(request_context.identity_policies) == 0: + if len(resource_based_allowed_scopes) == 0: + # nothing from resource based policies and no identity based policies -> implicit deny + return None + # we still have to check permission boundaries if there are any, go to step 5 + else: + identity_based_allowed = check_identity_based_policies(request_context, resource, action) + if len(identity_based_allowed): + return None + + # 5. check permission boundaries + permission_boundary_conditions: List[Json] = [] + if len(request_context.permission_boundaries) > 0: + permission_boundary_allowed = check_permission_boundaries(request_context, resource, action) + match permission_boundary_allowed: + case True: + pass + case False: + return None + case only_if_conditions: + permission_boundary_conditions.extend(only_if_conditions) + + + # 6. check for session policies + # we don't collect session principals and session policies, so this step is skipped + + # 7. if we reached here, the action is allowed + service, action_name = action.split(":") + action_data = get_action_data(service, action_name) + level = [info["access_level"] for info in action_data[service] if action == info["action"]][0] + + # 8. deduplicate the policies + + + # if there were any permission boundary conditions, we should merge them into the collected scopes + # todo: merge the boundary conditions into the scopes + + + # return the result + return AccessPermission( + action=action, + level=level, + scopes=resource_based_allowed_scopes + identity_based_scopes, # todo: add scopes + ) + + +def compute_permissions( + resource: AwsResource, + iam_context: IamRequestContext, + resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], +) -> List[AccessPermission]: + + # step 1: find the relevant action to check + relevant_actions = find_all_allowed_actions([p for _, p in iam_context.all_policies(resource_based_policies)]) + + all_permissions = [] + + # step 2: for every action, check if it is allowed + for action in relevant_actions: + if permissions := check_policies(iam_context, resource, action, resource_based_policies) + all_permissions.extend(permissions) + + return all_permissions + + + +class AccessEdgeCreator: + + def __init__(self, builder: GraphBuilder): + self.builder = builder + self.principals: List[IamRequestContext] = [] + + def init_principals(self) -> None: + for node in self.builder.nodes(clazz=AwsResource): + if isinstance(node, AwsIamUser): + + identity_based_policies = self.get_identity_based_policies(node) + permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this + service_control_policies: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this, see https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html + + request_context = IamRequestContext( + principal=node, + identity_policies=identity_based_policies, + permission_boundaries=permission_boundaries, + service_control_policies=service_control_policies, + ) + + self.principals.append(request_context) + + def get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[PolicySource, PolicyDocument]]: + if isinstance(principal, AwsIamUser): + inline_policies = [ + ( + PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn or ""), + PolicyDocument(policy.policy_document), + ) + for policy in principal.user_policies + if policy.policy_document + ] + attached_policies = [] + group_policies = [] + for _, to_node in self.builder.graph.edges(principal): + if isinstance(to_node, AwsIamPolicy): + if doc := find_policy_doc(to_node): + attached_policies.append( + ( + PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn or ""), + PolicyDocument(doc), + ) + ) + + if isinstance(to_node, AwsIamGroup): + group = to_node + # inline group policies + for policy in group.group_policies: + if policy.policy_document: + group_policies.append( + ( + PolicySource(kind=PolicySourceKind.Group, arn=group.arn or ""), + PolicyDocument(policy.policy_document), + ) + ) + # attached group policies + for _, group_successor in self.builder.graph.edges(group): + if isinstance(group_successor, AwsIamPolicy): + if doc := find_policy_doc(group_successor): + group_policies.append( + ( + PolicySource(kind=PolicySourceKind.Group, arn=group.arn or ""), + PolicyDocument(doc), + ) + ) + + return inline_policies + attached_policies + group_policies + + return [] + + + + def add_access_edges(self) -> None: + + for node in self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None): + for context in self.principals: + + resource_policies: List[Tuple[PolicySource, PolicyDocument]] = [] + if isinstance(node, HasResourcePolicy): + for source, json_policy in node.resource_policy(self.builder): + resource_policies.append((source, PolicyDocument(json_policy))) + + permissions = compute_permissions(node, context, resource_policies) + + self.builder.add_edge( + from_node=context.principal, edge_type=EdgeType.access, permissions=permissions, to_node=node + ) diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index a3f19e49c6..f4330eb659 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -2,10 +2,11 @@ import logging from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from typing import List, Type, Optional, ClassVar, Union, cast, Dict, Any +from typing import Any, Dict, List, Type, Optional, ClassVar, Union, cast, Dict, Any from attrs import define +from fix_plugin_aws.access_edges import AccessEdgeCreator from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.configuration import AwsConfig from fix_plugin_aws.resource import ( @@ -252,6 +253,10 @@ def get_last_run() -> Optional[datetime]: log.warning(f"Unexpected node type {node} in graph") raise Exception("Only AWS resources expected") + # add access edges + access_edge_creator = AccessEdgeCreator(global_builder) + access_edge_creator.add_access_edges() + # final hook when the graph is complete for node, data in list(self.graph.nodes(data=True)): if isinstance(node, AwsResource): diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index dd0dc591cf..c8517d2adc 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -15,6 +15,7 @@ from attrs import define from boto3.exceptions import Boto3Error +from fix_plugin_aws.access_edges import AccessPermission from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.configuration import AwsConfig from fix_plugin_aws.resource.pricing import AwsPricingPrice @@ -485,11 +486,20 @@ def node(self, clazz: Optional[Type[AwsResourceType]] = None, **node: Any) -> Op return n # type: ignore return None - def nodes(self, clazz: Optional[Type[AwsResourceType]] = None, **node: Any) -> Iterator[AwsResourceType]: + def nodes( + self, + clazz: Optional[Type[AwsResourceType]] = None, + filter: Optional[Callable[[AwsResourceType], bool]] = None, + **node: Any, + ) -> Iterator[AwsResourceType]: with self.graph_nodes_access.read_access: for n in self.graph: is_clazz = isinstance(n, clazz) if clazz else True - if is_clazz and all(getattr(n, k, None) == v for k, v in node.items()): + if ( + is_clazz + and (filter(n) if filter else True) + and all(getattr(n, k, None) == v for k, v in node.items()) + ): # noqa yield n def add_node( @@ -556,13 +566,18 @@ def add_node( return node def add_edge( - self, from_node: BaseResource, edge_type: EdgeType = EdgeType.default, reverse: bool = False, **to_node: Any + self, + from_node: BaseResource, + edge_type: EdgeType = EdgeType.default, + reverse: bool = False, + permissions: Optional[List[AccessPermission]] = None, + **to_node: Any, ) -> None: to_n = self.node(**to_node) if isinstance(from_node, AwsResource) and isinstance(to_n, AwsResource): start, end = (to_n, from_node) if reverse else (from_node, to_n) with self.graph_edges_access.write_access: - self.graph.add_edge(start, end, edge_type=edge_type) + self.graph.add_edge(start, end, edge_type=edge_type, permissions=permissions) def add_deferred_edge( self, from_node: BaseResource, edge_type: EdgeType, to_node: str, reverse: bool = False diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index ffb2b79b81..91f77e0d9c 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -2,11 +2,12 @@ from collections import defaultdict from datetime import timedelta from json import loads as json_loads -from typing import ClassVar, Dict, List, Type, Optional, cast, Any +from typing import ClassVar, Dict, List, Tuple, Type, Optional, cast, Any from attr import field from attrs import define +from fix_plugin_aws.access_edges import HasResourcePolicy, PolicySource from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder, parse_json from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory @@ -163,7 +164,7 @@ class AwsS3Logging: @define(eq=False, slots=False) -class AwsS3Bucket(AwsResource, BaseBucket): +class AwsS3Bucket(AwsResource, BaseBucket, HasResourcePolicy): kind: ClassVar[str] = "aws_s3_bucket" kind_display: ClassVar[str] = "AWS S3 Bucket" aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://s3.console.aws.amazon.com/s3/buckets/{name}?region={region_id}&bucketType=general&tab=objects", "arn_tpl": "arn:{partition}:s3:{region}:{account}:bucket/{name}"} # fmt: skip @@ -185,6 +186,9 @@ class AwsS3Bucket(AwsResource, BaseBucket): bucket_logging: Optional[AwsS3Logging] = field(default=None) bucket_location: Optional[str] = field(default=None) + def resource_policy(self, builder: GraphBuilder) -> List[Tuple[PolicySource, Dict[str, Any]]]: + return [(PolicySource.resource, self.bucket_policy)] if self.bucket_policy else [] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ From 816297ffd00dcad35e57e4256394df9df44df638 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 10:47:12 +0000 Subject: [PATCH 02/47] refactor the code a bit --- plugins/aws/fix_plugin_aws/access_edges.py | 467 ++++++++---------- .../aws/fix_plugin_aws/access_edges_utils.py | 37 ++ plugins/aws/fix_plugin_aws/resource/base.py | 2 +- plugins/aws/fix_plugin_aws/resource/s3.py | 5 +- plugins/aws/pyproject.toml | 1 + 5 files changed, 251 insertions(+), 261 deletions(-) create mode 100644 plugins/aws/fix_plugin_aws/access_edges_utils.py diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index b71afd0e01..fc90a92304 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -1,21 +1,24 @@ -from abc import ABC -from enum import Enum, StrEnum +from enum import Enum from functools import lru_cache from attr import define, evolve, frozen from fix_plugin_aws.resource.base import AwsResource, GraphBuilder -from typing import Dict, Any, List, Literal, Set, Optional, Tuple, TypedDict, Union, cast, Pattern +from typing import List, Literal, Set, Optional, Tuple, Union, Pattern +from fix_plugin_aws.access_edges_utils import ( + PolicySource, + PermissionScope, + AccessPermission, + PolicySourceKind, + HasResourcePolicy, +) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser -from fix_plugin_aws.resource.s3 import AwsS3Bucket -from fixcore.web import permission from fixlib.baseresources import EdgeType from fixlib.types import Json from cloudsplaining.scan.policy_document import PolicyDocument from cloudsplaining.scan.statement_detail import StatementDetail -from policy_sentry.querying.actions import get_action_data, get_actions_matching_arn -from policy_sentry.analysis.expand import determine_actions_to_expand +from policy_sentry.querying.actions import get_action_data from policy_sentry.querying.all import get_all_actions from policy_sentry.util.arns import ARN import re @@ -29,38 +32,6 @@ ResourceContstaint = str -class PolicySourceKind(StrEnum): - Principal = "principal" # e.g. IAM user, attached policy - Group = "group" # policy comes from an IAM group - Resource = "resource" # e.g. s3 bucket policy - - -class PolicySource(TypedDict): - kind: PolicySourceKind - arn: str - - -class HasResourcePolicy(ABC): - # returns a list of all policies that affects the resource (inline, attached, etc.) - def resource_policy(self, builder: GraphBuilder) -> List[Tuple[PolicySource, Json]]: - raise NotImplementedError - - - -@frozen -class PermissionScope: - source: PolicySource - restriction: str - conditions: List[Json] - - -@frozen -class AccessPermission: - action: str - level: str - scopes: List[PermissionScope] - - @define class IamRequestContext: principal: AwsResource @@ -94,7 +65,6 @@ def find_policy_doc(policy: AwsIamPolicy) -> Optional[Json]: def find_allowed_action(policy_document: PolicyDocument) -> Set[IamAction]: allowed_actions: Set[IamAction] = set() for statement in policy_document.statements: - statement = cast(StatementDetail, statement) if statement.effect_allow: allowed_actions.update(get_expanded_action(statement)) @@ -111,7 +81,7 @@ def find_all_allowed_actions(all_involved_policies: List[PolicyDocument]) -> Set @lru_cache(maxsize=1024) def get_expanded_action(statement: StatementDetail) -> Set[str]: actions = set() - expanded: List[str] = statement.expanded_actions or [] # type: ignore + expanded: List[str] = statement.expanded_actions or [] for action in expanded: actions.add(action) @@ -125,13 +95,14 @@ def make_resoruce_regex(aws_resorce_wildcard: str) -> Pattern[str]: # step 2: compile the regex return re.compile(f"^{python_regex}$", re.IGNORECASE) + def expand_wildcards_and_match(*, identifier: str, wildcard_string: str) -> bool: """ helper function to expand wildcards and match the identifier use case: match the resource constraint (wildcard) with the ARN - match the wildcard action with the specific action + match the wildcard action with the specific action """ pattern = make_resoruce_regex(wildcard_string) return pattern.match(identifier) is not None @@ -142,109 +113,30 @@ class Allowed: resource_constraint: str condition: Optional[Json] = None + @frozen class Denied: pass - AccessResult = Union[Allowed, Denied] - -def policy_matching_statement_exists(policy: PolicyDocument, effect: Literal["Allow", "Deny"], action: str, resource: AwsResource) -> Optional[Tuple[StatementDetail, Optional[ResourceContstaint]]]: - """ - check if a matching statement exists in the policy, returns boolean if exists or json in case there is a condition we can't resolve - """ - if resource.arn is None: - raise ValueError("Resource ARN is missing, go and fix the filtering logic") - - for statement in policy.statements: - statement = cast(StatementDetail, statement) - - # step 1: check if the effect matches - if statement.effect != effect: - # wrong effect, skip this statement - continue - - # step 2: check if the action matches - action_match = False - if statement.actions: - for a in statement.actions: - if expand_wildcards_and_match(identifier=action, wildcard_string=a): - action_match = True - break - else: - # if not_action is also missing, consider all actions allowed - action_match = True - for na in statement.not_action: - if expand_wildcards_and_match(identifier=action, wildcard_string=na): - action_match = False - break - if not action_match: - # action does not match, skip this statement - continue - - # step 3: check if the resource matches - matched_resource_constraint: Optional[ResourceContstaint] = None - resource_matches = False - if len(statement.resources) > 0: - for resource_constraint in statement.resources: - if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): - matched_resource_constraint = resource_constraint - resource_matches = True - break - elif len(statement.not_resource) > 0: - resource_matches = True - for not_resource_constraint in statement.not_resource: - if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): - resource_matches = False - break - matched_resource_constraint = "not " + not_resource_constraint - else: - # no Resource/NotResource specified, consider allowed - resource_matches = True - if not resource_matches: - # resource does not match, skip this statement - continue - - # step 4: check if there is a condition to care about - return (statement, matched_resource_constraint) - - return None - - -def check_principal_match(principal: AwsResource, aws_principal_list: List[str]) -> bool: - for aws_principal in aws_principal_list: - if aws_principal == "*": - return True - - if principal.arn == aws_principal: - return True - - if principal.id == aws_principal: - return True - - principal_arn = ARN - if principal_arn.account == aws_principal: # type: ignore - return True - - return False - - -def collect_matching_resource_statements(principal: AwsResource, resoruce_policy: PolicyDocument, action: str, resource: AwsResource) -> List[Tuple[StatementDetail, Optional[ResourceContstaint]]]: +def check_statement_match( + statement: StatementDetail, + effect: Optional[Literal["Allow", "Deny"]], + action: str, + resource: AwsResource, + principal: Optional[AwsResource], +) -> Tuple[bool, Optional[ResourceContstaint]]: """ - resoruce based policies contain principal field and need to be handled differently + check if a statement matches the given effect, action, resource and principal, returns boolean if there is a match and optional resource constraint (if there were any) """ - results: List[Tuple[StatementDetail, Optional[ResourceContstaint]]] = [] - if resource.arn is None: raise ValueError("Resource ARN is missing, go and fix the filtering logic") - for statement in resoruce_policy.statements: - statement = cast(StatementDetail, statement) - - # step 1: check the principal + # step 1: check the principal if provided + if principal: principal_match = False if policy_principal := statement.json.get("Principal", None): if policy_principal == "*": @@ -273,88 +165,154 @@ def collect_matching_resource_statements(principal: AwsResource, resoruce_policy if not principal_match: # principal does not match, we can shortcut here - continue + return False, None + # step 2: check if the effect matches + if effect: + if statement.effect != effect: + # wrong effect, skip this statement + return False, None + + # step 3: check if the action matches + action_match = False + if statement.actions: + for a in statement.actions: + if expand_wildcards_and_match(identifier=action, wildcard_string=a): + action_match = True + break + else: + # not_action + action_match = True + for na in statement.not_action: + if expand_wildcards_and_match(identifier=action, wildcard_string=na): + action_match = False + break + if not action_match: + # action does not match, skip this statement + return False, None + + # step 4: check if the resource matches + matched_resource_constraint: Optional[ResourceContstaint] = None + resource_matches = False + if len(statement.resources) > 0: + for resource_constraint in statement.resources: + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): + matched_resource_constraint = resource_constraint + resource_matches = True + break + elif len(statement.not_resource) > 0: + resource_matches = True + for not_resource_constraint in statement.not_resource: + if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): + resource_matches = False + break + matched_resource_constraint = "not " + not_resource_constraint + else: + # no Resource/NotResource specified, consider allowed + resource_matches = True + if not resource_matches: + # resource does not match, skip this statement + return False, None - # step 2: check the action - action_match = False - if statement.actions: - for a in statement.actions: - if expand_wildcards_and_match(identifier=action, wildcard_string=a): - action_match = True - break - else: - # if not_action is also missing, consider all actions allowed - action_match = True - for na in statement.not_action: - if expand_wildcards_and_match(identifier=action, wildcard_string=na): - action_match = False - break - if not action_match: - # action does not match, skip this statement - continue + # step 5: (we're not doing this yet) check if the condition matches + # here we just return the statement and condition checking is the responsibility of the caller + return (True, matched_resource_constraint) - # step 3: check if the resource matches - matched_resource_constraint: Optional[ResourceContstaint] = None - resource_matches = False - if len(statement.resources) > 0: - for resource_constraint in statement.resources: - - if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): - resource_matches = True - matched_resource_constraint = resource_constraint - break - elif len(statement.not_resource) > 0: - resource_matches = True - for not_resource_constraint in statement.not_resource: - if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): - resource_matches = False - break - matched_resource_constraint = "not " + not_resource_constraint - else: - # no Resource/NotResource specified, consider allowed - resource_matches = True - if not resource_matches: - # resource does not match, skip this statement - continue - results.append((statement, matched_resource_constraint)) +def policy_matching_statement_exists( + policy: PolicyDocument, effect: Literal["Allow", "Deny"], action: str, resource: AwsResource +) -> Optional[Tuple[StatementDetail, Optional[ResourceContstaint]]]: + """ + this function is used for policies that do not have principal field, e.g. all non-resource based policies + """ + if resource.arn is None: + raise ValueError("Resource ARN is missing, go and fix the filtering logic") + + for statement in policy.statements: + matches, maybe_resource_constraint = check_statement_match(statement, effect, action, resource, principal=None) + if matches: + return (statement, maybe_resource_constraint) - return results + return None + + +def check_principal_match(principal: AwsResource, aws_principal_list: List[str]) -> bool: + assert principal.arn + for aws_principal in aws_principal_list: + if aws_principal == "*": + return True + + if principal.arn == aws_principal: + return True + + if principal.id == aws_principal: + return True + principal_arn = ARN(principal.arn) + if principal_arn.account == aws_principal: + return True + + return False +def collect_matching_resource_statements( + principal: AwsResource, resoruce_policy: PolicyDocument, action: str, resource: AwsResource +) -> List[Tuple[StatementDetail, Optional[ResourceContstaint]]]: + """ + resoruce based policies contain principal field and need to be handled differently + """ + results: List[Tuple[StatementDetail, Optional[ResourceContstaint]]] = [] + + if resource.arn is None: + raise ValueError("Resource ARN is missing, go and fix the filtering logic") + + for statement in resoruce_policy.statements: + + matches, maybe_resource_constraint = check_statement_match( + statement, effect=None, action=action, resource=resource, principal=principal + ) + if matches: + results.append((statement, maybe_resource_constraint)) + + return results -def check_explicit_deny(request_context: IamRequestContext, resource: AwsResource, action: str, resource_based_policies: List[Tuple[PolicySource, PolicyDocument]]) -> Union[Literal['Denied', 'Allowed'], List[Json]]: +def check_explicit_deny( + request_context: IamRequestContext, + resource: AwsResource, + action: str, + resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], +) -> Union[Literal["Denied", "Allowed"], List[Json]]: denied_when_any: List[Json] = [] # check all the policies except the resource based ones for source, policy in request_context.all_policies(): - match policy_matching_statement_exists(policy, "Deny", action, resource): - case True: - return 'Denied' - case False: - pass - case condition: - denied_when_any.append(condition) + result = policy_matching_statement_exists(policy, "Deny", action, resource) + if result: + statement, resource_constraint = result + if statement.condition: + denied_when_any.append(statement.condition) + else: + return "Denied" for source, policy in resource_based_policies: # resource based policies require a different handling - resource_policy_statements = collect_matching_resource_statements(request_context.principal, policy, action, resource) + resource_policy_statements = collect_matching_resource_statements( + request_context.principal, policy, action, resource + ) for statement, _ in resource_policy_statements: if not statement.effect_deny: continue if statement.condition: denied_when_any.append(statement.condition) else: - return 'Denied' + return "Denied" if len(denied_when_any) > 0: return denied_when_any - return 'Allowed' + return "Allowed" def scp_allowed(request_context: IamRequestContext) -> bool: @@ -385,10 +343,11 @@ class Continue: # as a shortcut we return the first allow statement we find, or a first seen condition. # todo: collect all allow statements and conditions def check_resource_based_policies( - principal: AwsResource, - action: str, - resource: AwsResource, - resource_based_policies: List[Tuple[PolicySource, PolicyDocument]]) -> ResourceBasedPolicyResult: + principal: AwsResource, + action: str, + resource: AwsResource, + resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], +) -> ResourceBasedPolicyResult: scopes: List[PermissionScope] = [] @@ -402,31 +361,25 @@ def check_resource_based_policies( for statement, constraint in matching_statements: if statement.effect_allow: if statement.condition: - scopes.append(PermissionScope( - source=source, - restriction=constraint or "*", - conditions=[statement.condition] - )) + scopes.append( + PermissionScope(source=source, restriction=constraint or "*", conditions=[statement.condition]) + ) else: - scopes.append(PermissionScope( - source=source, - restriction=constraint or "*", - conditions=[] - )) + scopes.append(PermissionScope(source=source, restriction=constraint or "*", conditions=[])) - match principal.kind: - case "aws_iam_user": - # in case of IAM user, access is allowed regardless of the rest of the policies - return FinalAllow(scopes) + if isinstance(principal, AwsIamUser): + # in case of IAM user, access is allowed regardless of the rest of the policies + return FinalAllow(scopes) - # todo: deal with session principals somehow - case _: - # in case of other IAM principals, allow on resource based policy is not enough and - # we need to check the permission boundaries - return Continue(scopes) + # todo: deal with session principals somehow + # in case of other IAM principals, allow on resource based policy is not enough and + # we need to check the permission boundaries + return Continue(scopes) -def check_identity_based_policies(request_context: IamRequestContext, resource: AwsResource, action: str) -> List[PermissionScope]: +def check_identity_based_policies( + request_context: IamRequestContext, resource: AwsResource, action: str +) -> List[PermissionScope]: scopes: List[PermissionScope] = [] @@ -434,12 +387,15 @@ def check_identity_based_policies(request_context: IamRequestContext, resource: if exists := policy_matching_statement_exists(policy, "Allow", action, resource): statement, resource_constraint = exists restriction: str = resource_constraint or "*" + assert isinstance(statement.condition, dict) scopes.append(PermissionScope(source, restriction, [statement.condition])) return scopes -def check_permission_boundaries(request_context: IamRequestContext, resource: AwsResource, action: str) -> Union[bool, List[Json]]: +def check_permission_boundaries( + request_context: IamRequestContext, resource: AwsResource, action: str +) -> Union[bool, List[Json]]: conditions: List[Json] = [] @@ -453,15 +409,15 @@ def check_permission_boundaries(request_context: IamRequestContext, resource: Aw if len(conditions) > 0: return conditions - # no matching permission boundaries that allow access + # no matching permission boundaries that allow access return False - def negate_condition(condition: Json) -> Json: # todo: implement this return condition + def merge_conditions(conditions: List[Json]) -> Json: # todo: implement this return conditions[0] @@ -470,9 +426,10 @@ def merge_conditions(conditions: List[Json]) -> Json: def get_action_level(action: str) -> str: service, action_name = action.split(":") action_data = get_action_data(service, action_name) - level = [info["access_level"] for info in action_data[service] if action == info["action"]][0] + level: str = [info["access_level"] for info in action_data[service] if action == info["action"]][0] return level + # logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html#policy-eval-denyallow def check_policies( request_context: IamRequestContext, @@ -480,18 +437,18 @@ def check_policies( action: str, resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], ) -> Optional[AccessPermission]: - + deny_conditions: Optional[Json] = None # 1. check for explicit deny. If denied, we can abort immediately - match check_explicit_deny(request_context, resource, action, resource_based_policies): - case 'Denied': - return None - case 'Allowed': - pass - case conditions: - negated = [negate_condition(c) for c in conditions] - merged = merge_conditions(negated) - deny_conditions = merged + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + if result == "Denied": + return None + elif result == "Allowed": + pass + else: + negated = [negate_condition(c) for c in result] + merged = merge_conditions(negated) + deny_conditions = merged # 2. check for organization SCPs if len(request_context.service_control_policies) > 0: @@ -502,23 +459,22 @@ def check_policies( # 3. check resource based policies resource_based_allowed_scopes: List[PermissionScope] = [] if len(resource_based_policies) > 0: - match check_resource_based_policies(request_context.principal, action, resource, resource_based_policies): - case FinalAllow(scopes): - allowed_scopes: List[PermissionScope] = [] - for scope in scopes: - if deny_conditions: - new_conditions = [merge_conditions([deny_conditions, c]) for c in scope.conditions] - scope = evolve(scope, conditions=new_conditions) - allowed_scopes.append(scope) - - return AccessPermission( - action=action, - level=get_action_level(action), - scopes=allowed_scopes - ) - - case Continue(scopes): - resource_based_allowed_scopes.extend(scopes) + resource_result = check_resource_based_policies( + request_context.principal, action, resource, resource_based_policies + ) + if isinstance(resource_result, FinalAllow): + scopes = resource_result.scopes + allowed_scopes: List[PermissionScope] = [] + for scope in scopes: + if deny_conditions: + new_conditions = [merge_conditions([deny_conditions, c]) for c in scope.conditions] + scope = evolve(scope, conditions=new_conditions) + allowed_scopes.append(scope) + + return AccessPermission(action=action, level=get_action_level(action), scopes=allowed_scopes) + if isinstance(resource_result, Continue): + scopes = resource_result.scopes + resource_based_allowed_scopes.extend(scopes) # 4. check identity based policies identity_based_scopes: List[PermissionScope] = [] @@ -536,14 +492,12 @@ def check_policies( permission_boundary_conditions: List[Json] = [] if len(request_context.permission_boundaries) > 0: permission_boundary_allowed = check_permission_boundaries(request_context, resource, action) - match permission_boundary_allowed: - case True: - pass - case False: - return None - case only_if_conditions: - permission_boundary_conditions.extend(only_if_conditions) - + if permission_boundary_allowed is False: + return None + if permission_boundary_allowed is True: + pass + if isinstance(permission_boundary_allowed, list): + permission_boundary_conditions.extend(permission_boundary_allowed) # 6. check for session policies # we don't collect session principals and session policies, so this step is skipped @@ -555,11 +509,9 @@ def check_policies( # 8. deduplicate the policies - # if there were any permission boundary conditions, we should merge them into the collected scopes # todo: merge the boundary conditions into the scopes - # return the result return AccessPermission( action=action, @@ -577,17 +529,16 @@ def compute_permissions( # step 1: find the relevant action to check relevant_actions = find_all_allowed_actions([p for _, p in iam_context.all_policies(resource_based_policies)]) - all_permissions = [] + all_permissions: List[AccessPermission] = [] # step 2: for every action, check if it is allowed for action in relevant_actions: - if permissions := check_policies(iam_context, resource, action, resource_based_policies) - all_permissions.extend(permissions) + if p := check_policies(iam_context, resource, action, resource_based_policies): + all_permissions.append(p) return all_permissions - class AccessEdgeCreator: def __init__(self, builder: GraphBuilder): @@ -600,7 +551,9 @@ def init_principals(self) -> None: identity_based_policies = self.get_identity_based_policies(node) permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this - service_control_policies: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this, see https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html + service_control_policies: List[Tuple[PolicySource, PolicyDocument]] = ( + [] + ) # todo: add this, see https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html request_context = IamRequestContext( principal=node, @@ -659,8 +612,6 @@ def get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Poli return [] - - def add_access_edges(self) -> None: for node in self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None): diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py new file mode 100644 index 0000000000..ac8b1b08e7 --- /dev/null +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -0,0 +1,37 @@ +from enum import StrEnum +from abc import ABC +from attrs import frozen +from typing import List, Tuple, Any +from fixlib.types import Json + + +class PolicySourceKind(StrEnum): + Principal = "principal" # e.g. IAM user, attached policy + Group = "group" # policy comes from an IAM group + Resource = "resource" # e.g. s3 bucket policy + + +@frozen +class PolicySource: + kind: PolicySourceKind + arn: str + + +class HasResourcePolicy(ABC): + # returns a list of all policies that affects the resource (inline, attached, etc.) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: + raise NotImplementedError + + +@frozen +class PermissionScope: + source: PolicySource + restriction: str + conditions: List[Json] + + +@frozen +class AccessPermission: + action: str + level: str + scopes: List[PermissionScope] diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index c8517d2adc..f71dbd50a2 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -15,7 +15,7 @@ from attrs import define from boto3.exceptions import Boto3Error -from fix_plugin_aws.access_edges import AccessPermission +from fix_plugin_aws.access_edges_utils import AccessPermission from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.configuration import AwsConfig from fix_plugin_aws.resource.pricing import AwsPricingPrice diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index 91f77e0d9c..7c1146db3d 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -7,7 +7,7 @@ from attr import field from attrs import define -from fix_plugin_aws.access_edges import HasResourcePolicy, PolicySource +from fix_plugin_aws.access_edges_utils import HasResourcePolicy, PolicySource, PolicySourceKind from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder, parse_json from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory @@ -187,7 +187,8 @@ class AwsS3Bucket(AwsResource, BaseBucket, HasResourcePolicy): bucket_location: Optional[str] = field(default=None) def resource_policy(self, builder: GraphBuilder) -> List[Tuple[PolicySource, Dict[str, Any]]]: - return [(PolicySource.resource, self.bucket_policy)] if self.bucket_policy else [] + assert self.arn + return [(PolicySource(PolicySourceKind.Resource, self.arn), self.bucket_policy)] if self.bucket_policy else [] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/pyproject.toml b/plugins/aws/pyproject.toml index e20d85b39b..06811acc47 100644 --- a/plugins/aws/pyproject.toml +++ b/plugins/aws/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "retrying", "boto3", "botocore", + "cloudsplaining", ] [project.entry-points."fix.plugins"] From afc8d69887b2a4ae39ccef9e7b543c28206e000c Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 11:10:11 +0000 Subject: [PATCH 03/47] handle service linked roles when checking SCPs policies --- plugins/aws/fix_plugin_aws/access_edges.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index fc90a92304..462ddca8bd 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -286,8 +286,19 @@ def check_explicit_deny( denied_when_any: List[Json] = [] + # we should skip service control policies for service linked roles + if not service_linked_role(request_context.principal): + for source, policy in request_context.service_control_policies: + result = policy_matching_statement_exists(policy, "Deny", action, resource) + if result: + statement, resource_constraint = result + if statement.condition: + denied_when_any.append(statement.condition) + else: + return "Denied" + # check all the policies except the resource based ones - for source, policy in request_context.all_policies(): + for source, policy in request_context.identity_policies + request_context.permission_boundaries: result = policy_matching_statement_exists(policy, "Deny", action, resource) if result: statement, resource_constraint = result @@ -423,6 +434,11 @@ def merge_conditions(conditions: List[Json]) -> Json: return conditions[0] +def service_linked_role(principal: AwsResource) -> bool: + # todo: implement this + return False + + def get_action_level(action: str) -> str: service, action_name = action.split(":") action_data = get_action_data(service, action_name) @@ -451,7 +467,7 @@ def check_policies( deny_conditions = merged # 2. check for organization SCPs - if len(request_context.service_control_policies) > 0: + if len(request_context.service_control_policies) > 0 and not service_linked_role(request_context.principal): org_scp_allowed = scp_allowed(request_context) if not org_scp_allowed: return None From dd43128cbb44b47c3da1c5b219987c517cd769c6 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 11:12:27 +0000 Subject: [PATCH 04/47] fix the source in explicit deny check --- plugins/aws/fix_plugin_aws/access_edges.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 462ddca8bd..f5abea46f1 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -288,7 +288,7 @@ def check_explicit_deny( # we should skip service control policies for service linked roles if not service_linked_role(request_context.principal): - for source, policy in request_context.service_control_policies: + for _, policy in request_context.service_control_policies: result = policy_matching_statement_exists(policy, "Deny", action, resource) if result: statement, resource_constraint = result @@ -298,7 +298,7 @@ def check_explicit_deny( return "Denied" # check all the policies except the resource based ones - for source, policy in request_context.identity_policies + request_context.permission_boundaries: + for _, policy in request_context.identity_policies + request_context.permission_boundaries: result = policy_matching_statement_exists(policy, "Deny", action, resource) if result: statement, resource_constraint = result @@ -307,7 +307,7 @@ def check_explicit_deny( else: return "Denied" - for source, policy in resource_based_policies: + for _, policy in resource_based_policies: # resource based policies require a different handling resource_policy_statements = collect_matching_resource_statements( request_context.principal, policy, action, resource From 066a650ea6027ec47d8032a807d29cea519eebf8 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 11:25:59 +0000 Subject: [PATCH 05/47] implement basic scp control group checks --- plugins/aws/fix_plugin_aws/access_edges.py | 25 ++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index f5abea46f1..c8f6ffe82e 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -37,7 +37,7 @@ class IamRequestContext: principal: AwsResource identity_policies: List[Tuple[PolicySource, PolicyDocument]] permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too - service_control_policies: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too + service_control_groups: List[List[Tuple[PolicySource, PolicyDocument]]] # todo: use them too # technically we should also add a list of session policies here, but they don't exist in the collector context def all_policies( @@ -46,7 +46,7 @@ def all_policies( return ( self.identity_policies + self.permission_boundaries - + self.service_control_policies + + [p for group in self.service_control_groups for p in group] + (resource_based_policies or []) ) @@ -326,8 +326,21 @@ def check_explicit_deny( return "Allowed" -def scp_allowed(request_context: IamRequestContext) -> bool: - # todo: collect the necessary resources and implement this +def scp_allowed(request_context: IamRequestContext, action: str, resource: AwsResource) -> bool: + + for group in request_context.service_control_groups: + scp_group_allows = False + for _, policy in group: + if exists := policy_matching_statement_exists(policy, "Allow", action, resource): + statement, resource_constraint = exists + if not statement.condition: + scp_group_allows = True + break + # todo: handle scp conditions + + if not scp_group_allows: + return False + return True @@ -467,8 +480,8 @@ def check_policies( deny_conditions = merged # 2. check for organization SCPs - if len(request_context.service_control_policies) > 0 and not service_linked_role(request_context.principal): - org_scp_allowed = scp_allowed(request_context) + if len(request_context.service_control_groups) > 0 and not service_linked_role(request_context.principal): + org_scp_allowed = scp_allowed(request_context, action, resource) if not org_scp_allowed: return None From 4fd45ecd793086c55b7ef2fbcbdb28fa4b7e233a Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 13:08:51 +0000 Subject: [PATCH 06/47] evaluate SCPs correctly --- plugins/aws/fix_plugin_aws/access_edges.py | 97 +++++++++++----------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index c8f6ffe82e..55d5a6ee63 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -37,7 +37,8 @@ class IamRequestContext: principal: AwsResource identity_policies: List[Tuple[PolicySource, PolicyDocument]] permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too - service_control_groups: List[List[Tuple[PolicySource, PolicyDocument]]] # todo: use them too + # all service control policies applicable to the principal, starting from the root, then all org units, then the account + service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] # technically we should also add a list of session policies here, but they don't exist in the collector context def all_policies( @@ -46,7 +47,7 @@ def all_policies( return ( self.identity_policies + self.permission_boundaries - + [p for group in self.service_control_groups for p in group] + + [p for group in self.service_control_policy_levels for p in group] + (resource_based_policies or []) ) @@ -282,28 +283,29 @@ def check_explicit_deny( resource: AwsResource, action: str, resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], -) -> Union[Literal["Denied", "Allowed"], List[Json]]: +) -> Union[Literal["Denied", "NextStep"], List[Json]]: - denied_when_any: List[Json] = [] + denied_when_any_is_true: List[Json] = [] # we should skip service control policies for service linked roles - if not service_linked_role(request_context.principal): - for _, policy in request_context.service_control_policies: - result = policy_matching_statement_exists(policy, "Deny", action, resource) - if result: - statement, resource_constraint = result - if statement.condition: - denied_when_any.append(statement.condition) - else: - return "Denied" + if not is_service_linked_role(request_context.principal): + for scp_level in request_context.service_control_policy_levels: + for _, policy in scp_level: + result = policy_matching_statement_exists(policy, "Deny", action, resource) + if result: + statement, _ = result + if statement.condition: + denied_when_any_is_true.append(statement.condition) + else: + return "Denied" # check all the policies except the resource based ones for _, policy in request_context.identity_policies + request_context.permission_boundaries: result = policy_matching_statement_exists(policy, "Deny", action, resource) if result: - statement, resource_constraint = result + statement, _ = result if statement.condition: - denied_when_any.append(statement.condition) + denied_when_any_is_true.append(statement.condition) else: return "Denied" @@ -316,29 +318,29 @@ def check_explicit_deny( if not statement.effect_deny: continue if statement.condition: - denied_when_any.append(statement.condition) + denied_when_any_is_true.append(statement.condition) else: return "Denied" - if len(denied_when_any) > 0: - return denied_when_any + if len(denied_when_any_is_true) > 0: + return denied_when_any_is_true - return "Allowed" + return "NextStep" def scp_allowed(request_context: IamRequestContext, action: str, resource: AwsResource) -> bool: - for group in request_context.service_control_groups: - scp_group_allows = False - for _, policy in group: + # traverse the SCPs: root -> OU -> account levels + for scp_level_policies in request_context.service_control_policy_levels: + level_allows = False + for _, policy in scp_level_policies: if exists := policy_matching_statement_exists(policy, "Allow", action, resource): - statement, resource_constraint = exists - if not statement.condition: - scp_group_allows = True - break - # todo: handle scp conditions + # 'Allow' statement in SCP can't have conditions, so we can shortcut here + statement, _ = exists + level_allows = True + break - if not scp_group_allows: + if not level_allows: return False return True @@ -437,17 +439,7 @@ def check_permission_boundaries( return False -def negate_condition(condition: Json) -> Json: - # todo: implement this - return condition - - -def merge_conditions(conditions: List[Json]) -> Json: - # todo: implement this - return conditions[0] - - -def service_linked_role(principal: AwsResource) -> bool: +def is_service_linked_role(principal: AwsResource) -> bool: # todo: implement this return False @@ -467,20 +459,31 @@ def check_policies( resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], ) -> Optional[AccessPermission]: - deny_conditions: Optional[Json] = None + # when any of the conditions evaluate to true, the action is explicitly denied + # comes from any explicit deny statements in all policies + denying_conditions: List[Json] = [] + + # when any of the conditions evaluate to false, the action is implicitly denied + # comes from the permission boundaries + restricting_conditions: List[Json] = [] + + # when any of the conditions evaluate to true, the action is allowed (given it was not denied explicitly/implicitly before) + # comes from the resource based policies and identity based policies + allowing_conditions: List[Json] = [] + # 1. check for explicit deny. If denied, we can abort immediately result = check_explicit_deny(request_context, resource, action, resource_based_policies) if result == "Denied": return None - elif result == "Allowed": + elif result == "NextStep": pass else: - negated = [negate_condition(c) for c in result] - merged = merge_conditions(negated) - deny_conditions = merged + for c in result: + # satisfying any of the conditions above will deny the action + denying_conditions.append(c) # 2. check for organization SCPs - if len(request_context.service_control_groups) > 0 and not service_linked_role(request_context.principal): + if len(request_context.service_control_policy_levels) > 0 and not is_service_linked_role(request_context.principal): org_scp_allowed = scp_allowed(request_context, action, resource) if not org_scp_allowed: return None @@ -580,7 +583,7 @@ def init_principals(self) -> None: identity_based_policies = self.get_identity_based_policies(node) permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this - service_control_policies: List[Tuple[PolicySource, PolicyDocument]] = ( + service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] = ( [] ) # todo: add this, see https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html @@ -588,7 +591,7 @@ def init_principals(self) -> None: principal=node, identity_policies=identity_based_policies, permission_boundaries=permission_boundaries, - service_control_policies=service_control_policies, + service_control_policy_levels=service_control_policy_levels, ) self.principals.append(request_context) From f93e96dc201931f945c2b86847fa67a547ef0ded Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 14:32:56 +0000 Subject: [PATCH 07/47] adjust resource based policies --- plugins/aws/fix_plugin_aws/access_edges.py | 157 ++++++++++-------- .../aws/fix_plugin_aws/access_edges_utils.py | 8 +- 2 files changed, 90 insertions(+), 75 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 55d5a6ee63..4b07686549 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -129,7 +129,7 @@ def check_statement_match( action: str, resource: AwsResource, principal: Optional[AwsResource], -) -> Tuple[bool, Optional[ResourceContstaint]]: +) -> Tuple[bool, List[ResourceContstaint]]: """ check if a statement matches the given effect, action, resource and principal, returns boolean if there is a match and optional resource constraint (if there were any) """ @@ -166,13 +166,13 @@ def check_statement_match( if not principal_match: # principal does not match, we can shortcut here - return False, None + return False, [] # step 2: check if the effect matches if effect: if statement.effect != effect: # wrong effect, skip this statement - return False, None + return False, [] # step 3: check if the action matches action_match = False @@ -190,15 +190,15 @@ def check_statement_match( break if not action_match: # action does not match, skip this statement - return False, None + return False, [] # step 4: check if the resource matches - matched_resource_constraint: Optional[ResourceContstaint] = None + matched_resource_constraints: List[ResourceContstaint] = [] resource_matches = False if len(statement.resources) > 0: for resource_constraint in statement.resources: if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=resource_constraint): - matched_resource_constraint = resource_constraint + matched_resource_constraints.append(resource_constraint) resource_matches = True break elif len(statement.not_resource) > 0: @@ -207,30 +207,37 @@ def check_statement_match( if expand_wildcards_and_match(identifier=resource.arn, wildcard_string=not_resource_constraint): resource_matches = False break - matched_resource_constraint = "not " + not_resource_constraint + matched_resource_constraints.append("not " + not_resource_constraint) else: # no Resource/NotResource specified, consider allowed resource_matches = True if not resource_matches: # resource does not match, skip this statement - return False, None + return False, [] # step 5: (we're not doing this yet) check if the condition matches # here we just return the statement and condition checking is the responsibility of the caller - return (True, matched_resource_constraint) + return (True, matched_resource_constraints) def policy_matching_statement_exists( - policy: PolicyDocument, effect: Literal["Allow", "Deny"], action: str, resource: AwsResource -) -> Optional[Tuple[StatementDetail, Optional[ResourceContstaint]]]: + policy: PolicyDocument, + effect: Literal["Allow", "Deny"], + action: str, + resource: AwsResource, + *, + principal: Optional[AwsResource] = None, +) -> Optional[Tuple[StatementDetail, List[ResourceContstaint]]]: """ - this function is used for policies that do not have principal field, e.g. all non-resource based policies + only use this when we don't care about the conditions """ if resource.arn is None: raise ValueError("Resource ARN is missing, go and fix the filtering logic") for statement in policy.statements: - matches, maybe_resource_constraint = check_statement_match(statement, effect, action, resource, principal=None) + matches, maybe_resource_constraint = check_statement_match( + statement, effect, action, resource, principal=principal + ) if matches: return (statement, maybe_resource_constraint) @@ -256,21 +263,26 @@ def check_principal_match(principal: AwsResource, aws_principal_list: List[str]) return False -def collect_matching_resource_statements( - principal: AwsResource, resoruce_policy: PolicyDocument, action: str, resource: AwsResource -) -> List[Tuple[StatementDetail, Optional[ResourceContstaint]]]: +def collect_matching_statements( + *, + policy: PolicyDocument, + effect: Optional[Literal["Allow", "Deny"]], + action: str, + resource: AwsResource, + principal: Optional[AwsResource], +) -> List[Tuple[StatementDetail, List[ResourceContstaint]]]: """ resoruce based policies contain principal field and need to be handled differently """ - results: List[Tuple[StatementDetail, Optional[ResourceContstaint]]] = [] + results: List[Tuple[StatementDetail, List[ResourceContstaint]]] = [] if resource.arn is None: raise ValueError("Resource ARN is missing, go and fix the filtering logic") - for statement in resoruce_policy.statements: + for statement in policy.statements: matches, maybe_resource_constraint = check_statement_match( - statement, effect=None, action=action, resource=resource, principal=principal + statement, effect=effect, action=action, resource=resource, principal=principal ) if matches: results.append((statement, maybe_resource_constraint)) @@ -291,38 +303,29 @@ def check_explicit_deny( if not is_service_linked_role(request_context.principal): for scp_level in request_context.service_control_policy_levels: for _, policy in scp_level: - result = policy_matching_statement_exists(policy, "Deny", action, resource) - if result: - statement, _ = result + policy_statements = collect_matching_statements( + policy=policy, effect="Deny", action=action, resource=resource, principal=request_context.principal + ) + for statement, _ in policy_statements: if statement.condition: denied_when_any_is_true.append(statement.condition) else: return "Denied" - # check all the policies except the resource based ones - for _, policy in request_context.identity_policies + request_context.permission_boundaries: - result = policy_matching_statement_exists(policy, "Deny", action, resource) - if result: - statement, _ = result - if statement.condition: - denied_when_any_is_true.append(statement.condition) - else: - return "Denied" - - for _, policy in resource_based_policies: - # resource based policies require a different handling - resource_policy_statements = collect_matching_resource_statements( - request_context.principal, policy, action, resource + # check the rest of the policies + for _, policy in ( + request_context.identity_policies + request_context.permission_boundaries + resource_based_policies + ): + policy_statements = collect_matching_statements( + policy=policy, effect="Deny", action=action, resource=resource, principal=request_context.principal ) - for statement, _ in resource_policy_statements: - if not statement.effect_deny: - continue + for statement, _ in policy_statements: if statement.condition: denied_when_any_is_true.append(statement.condition) else: return "Denied" - if len(denied_when_any_is_true) > 0: + if denied_when_any_is_true: return denied_when_any_is_true return "NextStep" @@ -334,9 +337,11 @@ def scp_allowed(request_context: IamRequestContext, action: str, resource: AwsRe for scp_level_policies in request_context.service_control_policy_levels: level_allows = False for _, policy in scp_level_policies: - if exists := policy_matching_statement_exists(policy, "Allow", action, resource): - # 'Allow' statement in SCP can't have conditions, so we can shortcut here - statement, _ = exists + statements = collect_matching_statements( + policy=policy, effect="Allow", action=action, resource=resource, principal=None + ) + if statements: + # 'Allow' statements in SCP can't have conditions, we do not check them level_allows = True break @@ -374,30 +379,41 @@ def check_resource_based_policies( resource: AwsResource, resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], ) -> ResourceBasedPolicyResult: + assert resource.arn scopes: List[PermissionScope] = [] - # todo: add special handling for IAM and KMS resources + # todo: support cross-account access evaluation + + arn = ARN(resource.arn) + if arn.service == "iam" or arn.service == "kms": # type: ignore + pass + # todo: implement implicit deny here for source, policy in resource_based_policies: - matching_statements = collect_matching_resource_statements(principal, policy, action, resource) + + matching_statements = collect_matching_statements( + policy=policy, + effect="Allow", + action=action, + resource=resource, + principal=principal, + ) if len(matching_statements) == 0: continue - for statement, constraint in matching_statements: - if statement.effect_allow: - if statement.condition: - scopes.append( - PermissionScope(source=source, restriction=constraint or "*", conditions=[statement.condition]) - ) - else: - scopes.append(PermissionScope(source=source, restriction=constraint or "*", conditions=[])) + for statement, constraints in matching_statements: + if statement.condition: + scopes.append(PermissionScope(source=source, constraints=constraints, conditions=[statement.condition])) + else: + scopes.append(PermissionScope(source=source, constraints=constraints, conditions=[])) - if isinstance(principal, AwsIamUser): - # in case of IAM user, access is allowed regardless of the rest of the policies - return FinalAllow(scopes) + if scopes: + if isinstance(principal, AwsIamUser): + # in case of IAM users, identity_based_policies and permission boundaries are not relevant + # and we can return the result immediately + return FinalAllow(scopes) - # todo: deal with session principals somehow # in case of other IAM principals, allow on resource based policy is not enough and # we need to check the permission boundaries return Continue(scopes) @@ -411,10 +427,9 @@ def check_identity_based_policies( for source, policy in request_context.identity_policies: if exists := policy_matching_statement_exists(policy, "Allow", action, resource): - statement, resource_constraint = exists - restriction: str = resource_constraint or "*" + statement, resource_constraints = exists assert isinstance(statement.condition, dict) - scopes.append(PermissionScope(source, restriction, [statement.condition])) + scopes.append(PermissionScope(source, resource_constraints, [statement.condition])) return scopes @@ -461,15 +476,15 @@ def check_policies( # when any of the conditions evaluate to true, the action is explicitly denied # comes from any explicit deny statements in all policies - denying_conditions: List[Json] = [] + deny_conditions: List[Json] = [] # when any of the conditions evaluate to false, the action is implicitly denied # comes from the permission boundaries restricting_conditions: List[Json] = [] - # when any of the conditions evaluate to true, the action is allowed (given it was not denied explicitly/implicitly before) + # when any of the scopes evaluate to true, the action is allowed # comes from the resource based policies and identity based policies - allowing_conditions: List[Json] = [] + allowed_scopes: List[PermissionScope] = [] # 1. check for explicit deny. If denied, we can abort immediately result = check_explicit_deny(request_context, resource, action, resource_based_policies) @@ -480,7 +495,7 @@ def check_policies( else: for c in result: # satisfying any of the conditions above will deny the action - denying_conditions.append(c) + deny_conditions.append(c) # 2. check for organization SCPs if len(request_context.service_control_policy_levels) > 0 and not is_service_linked_role(request_context.principal): @@ -489,29 +504,25 @@ def check_policies( return None # 3. check resource based policies - resource_based_allowed_scopes: List[PermissionScope] = [] if len(resource_based_policies) > 0: resource_result = check_resource_based_policies( request_context.principal, action, resource, resource_based_policies ) if isinstance(resource_result, FinalAllow): scopes = resource_result.scopes - allowed_scopes: List[PermissionScope] = [] + final_scopes: List[PermissionScope] = [] for scope in scopes: - if deny_conditions: - new_conditions = [merge_conditions([deny_conditions, c]) for c in scope.conditions] - scope = evolve(scope, conditions=new_conditions) - allowed_scopes.append(scope) + final_scopes.append(scope.with_deny_conditions(deny_conditions)) - return AccessPermission(action=action, level=get_action_level(action), scopes=allowed_scopes) + return AccessPermission(action=action, level=get_action_level(action), scopes=final_scopes) if isinstance(resource_result, Continue): scopes = resource_result.scopes - resource_based_allowed_scopes.extend(scopes) + allowed_scopes.extend(scopes) # 4. check identity based policies identity_based_scopes: List[PermissionScope] = [] if len(request_context.identity_policies) == 0: - if len(resource_based_allowed_scopes) == 0: + if len(allowed_scopes) == 0: # nothing from resource based policies and no identity based policies -> implicit deny return None # we still have to check permission boundaries if there are any, go to step 5 diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py index ac8b1b08e7..e89aa5b599 100644 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -26,8 +26,12 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: @frozen class PermissionScope: source: PolicySource - restriction: str - conditions: List[Json] + constraints: List[str] # aka resource constraints + conditions: List[Json] # if nonempty and any is true, access is granted + deny_conditions: List[Json] # if nonempty and any is true, access is denied + + def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": + return PermissionScope(self.source, self.constraints, self.conditions, deny_conditions) @frozen From 93df70ac6cad2ae2c57aa2ee81d0afaa2af69366 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 15:19:29 +0000 Subject: [PATCH 08/47] fix how identity-based policies work --- plugins/aws/fix_plugin_aws/access_edges.py | 93 ++++++++++--------- .../aws/fix_plugin_aws/access_edges_utils.py | 16 +++- 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 4b07686549..3929ebbaa8 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -1,6 +1,6 @@ from enum import Enum from functools import lru_cache -from attr import define, evolve, frozen +from attr import define, frozen from fix_plugin_aws.resource.base import AwsResource, GraphBuilder from typing import List, Literal, Set, Optional, Tuple, Union, Pattern @@ -11,6 +11,7 @@ AccessPermission, PolicySourceKind, HasResourcePolicy, + ResourceConstraint, ) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser from fixlib.baseresources import EdgeType @@ -29,8 +30,6 @@ ALL_ACTIONS = get_all_actions() -ResourceContstaint = str - @define class IamRequestContext: @@ -129,7 +128,7 @@ def check_statement_match( action: str, resource: AwsResource, principal: Optional[AwsResource], -) -> Tuple[bool, List[ResourceContstaint]]: +) -> Tuple[bool, List[ResourceConstraint]]: """ check if a statement matches the given effect, action, resource and principal, returns boolean if there is a match and optional resource constraint (if there were any) """ @@ -193,7 +192,7 @@ def check_statement_match( return False, [] # step 4: check if the resource matches - matched_resource_constraints: List[ResourceContstaint] = [] + matched_resource_constraints: List[ResourceConstraint] = [] resource_matches = False if len(statement.resources) > 0: for resource_constraint in statement.resources: @@ -227,7 +226,7 @@ def policy_matching_statement_exists( resource: AwsResource, *, principal: Optional[AwsResource] = None, -) -> Optional[Tuple[StatementDetail, List[ResourceContstaint]]]: +) -> Optional[Tuple[StatementDetail, List[ResourceConstraint]]]: """ only use this when we don't care about the conditions """ @@ -270,11 +269,11 @@ def collect_matching_statements( action: str, resource: AwsResource, principal: Optional[AwsResource], -) -> List[Tuple[StatementDetail, List[ResourceContstaint]]]: +) -> List[Tuple[StatementDetail, List[ResourceConstraint]]]: """ resoruce based policies contain principal field and need to be handled differently """ - results: List[Tuple[StatementDetail, List[ResourceContstaint]]] = [] + results: List[Tuple[StatementDetail, List[ResourceConstraint]]] = [] if resource.arn is None: raise ValueError("Resource ARN is missing, go and fix the filtering logic") @@ -404,10 +403,13 @@ def check_resource_based_policies( for statement, constraints in matching_statements: if statement.condition: - scopes.append(PermissionScope(source=source, constraints=constraints, conditions=[statement.condition])) + scopes.append( + PermissionScope(source=source, constraints=constraints, allow_conditions=[statement.condition]) + ) else: - scopes.append(PermissionScope(source=source, constraints=constraints, conditions=[])) + scopes.append(PermissionScope(source=source, constraints=constraints, allow_conditions=[])) + # if we found any allow statements, let's check the principal and act accordingly if scopes: if isinstance(principal, AwsIamUser): # in case of IAM users, identity_based_policies and permission boundaries are not relevant @@ -426,8 +428,9 @@ def check_identity_based_policies( scopes: List[PermissionScope] = [] for source, policy in request_context.identity_policies: - if exists := policy_matching_statement_exists(policy, "Allow", action, resource): - statement, resource_constraints = exists + for statement, resource_constraints in collect_matching_statements( + policy=policy, effect="Allow", action=action, resource=resource, principal=None + ): assert isinstance(statement.condition, dict) scopes.append(PermissionScope(source, resource_constraints, [statement.condition])) @@ -436,22 +439,27 @@ def check_identity_based_policies( def check_permission_boundaries( request_context: IamRequestContext, resource: AwsResource, action: str -) -> Union[bool, List[Json]]: +) -> Union[Literal["Denied", "NextStep"], List[Json]]: conditions: List[Json] = [] - for source, policy in request_context.permission_boundaries: - if exists := policy_matching_statement_exists(policy, "Allow", action, resource): - statement, constraint = exists - if not statement.condition: - return True - conditions.append(statement.condition) + # ignore policy sources and resource constraints because permission boundaries + # can never allow access to a resource, only restrict it + for _, policy in request_context.permission_boundaries: + for statement, _ in collect_matching_statements( + policy=policy, effect="Allow", action=action, resource=resource, principal=None + ): + if statement.condition: + assert isinstance(statement.condition, dict) + conditions.append(statement.condition) + else: # if there is an allow statement without a condition, the action is allowed + return "NextStep" if len(conditions) > 0: return conditions # no matching permission boundaries that allow access - return False + return "Denied" def is_service_linked_role(principal: AwsResource) -> bool: @@ -510,38 +518,36 @@ def check_policies( ) if isinstance(resource_result, FinalAllow): scopes = resource_result.scopes - final_scopes: List[PermissionScope] = [] + final_resource_scopes: List[PermissionScope] = [] for scope in scopes: - final_scopes.append(scope.with_deny_conditions(deny_conditions)) + final_resource_scopes.append(scope.with_deny_conditions(deny_conditions)) - return AccessPermission(action=action, level=get_action_level(action), scopes=final_scopes) + return AccessPermission(action=action, level=get_action_level(action), scopes=final_resource_scopes) if isinstance(resource_result, Continue): scopes = resource_result.scopes allowed_scopes.extend(scopes) - # 4. check identity based policies - identity_based_scopes: List[PermissionScope] = [] + # 4. to make it a bit simpler, we check the permission boundaries before checking identity based policies + if len(request_context.permission_boundaries) > 0: + permission_boundary_result = check_permission_boundaries(request_context, resource, action) + if permission_boundary_result == "Denied": + return None + elif permission_boundary_result == "NextStep": + pass + else: + restricting_conditions.extend(permission_boundary_result) + + # 5. check identity based policies if len(request_context.identity_policies) == 0: if len(allowed_scopes) == 0: - # nothing from resource based policies and no identity based policies -> implicit deny + # resource policy did no allow any actions and we have zero identity based policies -> implicit deny return None - # we still have to check permission boundaries if there are any, go to step 5 + # otherwise continue with the resource based policies else: identity_based_allowed = check_identity_based_policies(request_context, resource, action) - if len(identity_based_allowed): + if not identity_based_allowed: return None - # 5. check permission boundaries - permission_boundary_conditions: List[Json] = [] - if len(request_context.permission_boundaries) > 0: - permission_boundary_allowed = check_permission_boundaries(request_context, resource, action) - if permission_boundary_allowed is False: - return None - if permission_boundary_allowed is True: - pass - if isinstance(permission_boundary_allowed, list): - permission_boundary_conditions.extend(permission_boundary_allowed) - # 6. check for session policies # we don't collect session principals and session policies, so this step is skipped @@ -550,16 +556,15 @@ def check_policies( action_data = get_action_data(service, action_name) level = [info["access_level"] for info in action_data[service] if action == info["action"]][0] - # 8. deduplicate the policies - - # if there were any permission boundary conditions, we should merge them into the collected scopes - # todo: merge the boundary conditions into the scopes + final_scopes: List[PermissionScope] = [] + for scope in allowed_scopes: + final_scopes.append(scope.with_deny_conditions(deny_conditions)) # return the result return AccessPermission( action=action, level=level, - scopes=resource_based_allowed_scopes + identity_based_scopes, # todo: add scopes + scopes=final_scopes, ) diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py index e89aa5b599..e3b266e1da 100644 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -1,9 +1,11 @@ from enum import StrEnum from abc import ABC -from attrs import frozen +from attrs import frozen, evolve from typing import List, Tuple, Any from fixlib.types import Json +ResourceConstraint = str + class PolicySourceKind(StrEnum): Principal = "principal" # e.g. IAM user, attached policy @@ -26,12 +28,16 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: @frozen class PermissionScope: source: PolicySource - constraints: List[str] # aka resource constraints - conditions: List[Json] # if nonempty and any is true, access is granted - deny_conditions: List[Json] # if nonempty and any is true, access is denied + constraints: List[ResourceConstraint] # aka resource constraints + allow_conditions: List[Json] # if nonempty and any evals to true, access is granted, otherwise implicitly denied + boundary_conditions: List[Json] = [] # if nonempty and any is evals to false, access is implicitly denied + deny_conditions: List[Json] = [] # if nonempty and any evals to true, access is explicitly denied def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": - return PermissionScope(self.source, self.constraints, self.conditions, deny_conditions) + return evolve(self, deny_conditions=deny_conditions) + + def with_boundary_conditions(self, boundary_conditions: List[Json]) -> "PermissionScope": + return evolve(self, boundary_conditions=boundary_conditions) @frozen From 9bbd05fa95222b336719118a13ca708575bc0e4b Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 18 Sep 2024 15:33:13 +0000 Subject: [PATCH 09/47] cleanup --- plugins/aws/fix_plugin_aws/access_edges.py | 69 +++------------------- plugins/aws/fix_plugin_aws/resource/iam.py | 3 + 2 files changed, 12 insertions(+), 60 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 3929ebbaa8..97b0f2004e 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -1,4 +1,3 @@ -from enum import Enum from functools import lru_cache from attr import define, frozen from fix_plugin_aws.resource.base import AwsResource, GraphBuilder @@ -36,7 +35,8 @@ class IamRequestContext: principal: AwsResource identity_policies: List[Tuple[PolicySource, PolicyDocument]] permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too - # all service control policies applicable to the principal, starting from the root, then all org units, then the account + # all service control policies applicable to the principal, + # starting from the root, then all org units, then the account service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] # technically we should also add a list of session policies here, but they don't exist in the collector context @@ -51,13 +51,6 @@ def all_policies( ) -def find_policy_doc(policy: AwsIamPolicy) -> Optional[Json]: - if not policy.policy_document: - return None - - return policy.policy_document.document - - IamAction = str @@ -108,20 +101,6 @@ def expand_wildcards_and_match(*, identifier: str, wildcard_string: str) -> bool return pattern.match(identifier) is not None -@frozen -class Allowed: - resource_constraint: str - condition: Optional[Json] = None - - -@frozen -class Denied: - pass - - -AccessResult = Union[Allowed, Denied] - - def check_statement_match( statement: StatementDetail, effect: Optional[Literal["Allow", "Deny"]], @@ -130,7 +109,8 @@ def check_statement_match( principal: Optional[AwsResource], ) -> Tuple[bool, List[ResourceConstraint]]: """ - check if a statement matches the given effect, action, resource and principal, returns boolean if there is a match and optional resource constraint (if there were any) + check if a statement matches the given effect, action, resource and principal, + returns boolean if there is a match and optional resource constraint (if there were any) """ if resource.arn is None: raise ValueError("Resource ARN is missing, go and fix the filtering logic") @@ -219,30 +199,6 @@ def check_statement_match( return (True, matched_resource_constraints) -def policy_matching_statement_exists( - policy: PolicyDocument, - effect: Literal["Allow", "Deny"], - action: str, - resource: AwsResource, - *, - principal: Optional[AwsResource] = None, -) -> Optional[Tuple[StatementDetail, List[ResourceConstraint]]]: - """ - only use this when we don't care about the conditions - """ - if resource.arn is None: - raise ValueError("Resource ARN is missing, go and fix the filtering logic") - - for statement in policy.statements: - matches, maybe_resource_constraint = check_statement_match( - statement, effect, action, resource, principal=principal - ) - if matches: - return (statement, maybe_resource_constraint) - - return None - - def check_principal_match(principal: AwsResource, aws_principal_list: List[str]) -> bool: assert principal.arn for aws_principal in aws_principal_list: @@ -350,12 +306,6 @@ def scp_allowed(request_context: IamRequestContext, action: str, resource: AwsRe return True -class ResourcePolicyCheckResult(Enum): - NO_MATCH = 0 - DENY_MATCH = 1 - ALLOW_MATCH = 2 - - @frozen class FinalAllow: scopes: List[PermissionScope] @@ -474,7 +424,7 @@ def get_action_level(action: str) -> str: return level -# logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html#policy-eval-denyallow +# logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html def check_policies( request_context: IamRequestContext, resource: AwsResource, @@ -599,9 +549,8 @@ def init_principals(self) -> None: identity_based_policies = self.get_identity_based_policies(node) permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this - service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] = ( - [] - ) # todo: add this, see https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html + # todo: https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html + service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] = [] request_context = IamRequestContext( principal=node, @@ -626,7 +575,7 @@ def get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Poli group_policies = [] for _, to_node in self.builder.graph.edges(principal): if isinstance(to_node, AwsIamPolicy): - if doc := find_policy_doc(to_node): + if doc := to_node.policy_document_json(): attached_policies.append( ( PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn or ""), @@ -648,7 +597,7 @@ def get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Poli # attached group policies for _, group_successor in self.builder.graph.edges(group): if isinstance(group_successor, AwsIamPolicy): - if doc := find_policy_doc(group_successor): + if doc := group_successor.policy_document_json(): group_policies.append( ( PolicySource(kind=PolicySourceKind.Group, arn=group.arn or ""), diff --git a/plugins/aws/fix_plugin_aws/resource/iam.py b/plugins/aws/fix_plugin_aws/resource/iam.py index 35096b7443..6e509052aa 100644 --- a/plugins/aws/fix_plugin_aws/resource/iam.py +++ b/plugins/aws/fix_plugin_aws/resource/iam.py @@ -387,6 +387,9 @@ def called_mutator_apis(cls) -> List[AwsApiSpec]: def service_name(cls) -> str: return service_name + def policy_document_json(self) -> Optional[Json]: + return self.policy_document.document if self.policy_document else None + @define(eq=False, slots=False) class AwsIamGroup(AwsResource, BaseGroup): From 802bc427dc60e1089294df4e4caf3bcf5346f1cf Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 20 Sep 2024 13:27:29 +0000 Subject: [PATCH 10/47] add explicit some more tests --- plugins/aws/fix_plugin_aws/access_edges.py | 40 ++- plugins/aws/test/acccess_edges_test.py | 378 +++++++++++++++++++++ 2 files changed, 403 insertions(+), 15 deletions(-) create mode 100644 plugins/aws/test/acccess_edges_test.py diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 97b0f2004e..70fe87da5f 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -34,20 +34,20 @@ class IamRequestContext: principal: AwsResource identity_policies: List[Tuple[PolicySource, PolicyDocument]] - permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] # todo: use them too + permission_boundaries: List[PolicyDocument] # todo: use them too # all service control policies applicable to the principal, # starting from the root, then all org units, then the account - service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] + service_control_policy_levels: List[List[PolicyDocument]] # technically we should also add a list of session policies here, but they don't exist in the collector context def all_policies( self, resource_based_policies: Optional[List[Tuple[PolicySource, PolicyDocument]]] = None - ) -> List[Tuple[PolicySource, PolicyDocument]]: + ) -> List[PolicyDocument]: return ( - self.identity_policies + [p[1] for p in self.identity_policies] + self.permission_boundaries + [p for group in self.service_control_policy_levels for p in group] - + (resource_based_policies or []) + + ([p[1] for p in (resource_based_policies or [])]) ) @@ -123,7 +123,8 @@ def check_statement_match( principal_match = True elif "AWS" in policy_principal: aws_principal_list = policy_principal["AWS"] - assert isinstance(aws_principal_list, list) + if isinstance(aws_principal_list, str): + aws_principal_list = [aws_principal_list] if check_principal_match(principal, aws_principal_list): principal_match = True else: @@ -257,7 +258,7 @@ def check_explicit_deny( # we should skip service control policies for service linked roles if not is_service_linked_role(request_context.principal): for scp_level in request_context.service_control_policy_levels: - for _, policy in scp_level: + for policy in scp_level: policy_statements = collect_matching_statements( policy=policy, effect="Deny", action=action, resource=resource, principal=request_context.principal ) @@ -267,10 +268,19 @@ def check_explicit_deny( else: return "Denied" + # check permission boundaries + for policy in request_context.permission_boundaries: + policy_statements = collect_matching_statements( + policy=policy, effect="Deny", action=action, resource=resource, principal=request_context.principal + ) + for statement, _ in policy_statements: + if statement.condition: + denied_when_any_is_true.append(statement.condition) + else: + return "Denied" + # check the rest of the policies - for _, policy in ( - request_context.identity_policies + request_context.permission_boundaries + resource_based_policies - ): + for _, policy in request_context.identity_policies + resource_based_policies: policy_statements = collect_matching_statements( policy=policy, effect="Deny", action=action, resource=resource, principal=request_context.principal ) @@ -291,7 +301,7 @@ def scp_allowed(request_context: IamRequestContext, action: str, resource: AwsRe # traverse the SCPs: root -> OU -> account levels for scp_level_policies in request_context.service_control_policy_levels: level_allows = False - for _, policy in scp_level_policies: + for policy in scp_level_policies: statements = collect_matching_statements( policy=policy, effect="Allow", action=action, resource=resource, principal=None ) @@ -395,7 +405,7 @@ def check_permission_boundaries( # ignore policy sources and resource constraints because permission boundaries # can never allow access to a resource, only restrict it - for _, policy in request_context.permission_boundaries: + for policy in request_context.permission_boundaries: for statement, _ in collect_matching_statements( policy=policy, effect="Allow", action=action, resource=resource, principal=None ): @@ -525,7 +535,7 @@ def compute_permissions( ) -> List[AccessPermission]: # step 1: find the relevant action to check - relevant_actions = find_all_allowed_actions([p for _, p in iam_context.all_policies(resource_based_policies)]) + relevant_actions = find_all_allowed_actions(iam_context.all_policies(resource_based_policies)) all_permissions: List[AccessPermission] = [] @@ -548,9 +558,9 @@ def init_principals(self) -> None: if isinstance(node, AwsIamUser): identity_based_policies = self.get_identity_based_policies(node) - permission_boundaries: List[Tuple[PolicySource, PolicyDocument]] = [] # todo: add this + permission_boundaries: List[PolicyDocument] = [] # todo: add this # todo: https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html - service_control_policy_levels: List[List[Tuple[PolicySource, PolicyDocument]]] = [] + service_control_policy_levels: List[List[PolicyDocument]] = [] request_context = IamRequestContext( principal=node, diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py new file mode 100644 index 0000000000..179b4c6cbb --- /dev/null +++ b/plugins/aws/test/acccess_edges_test.py @@ -0,0 +1,378 @@ +from cloudsplaining.scan.policy_document import PolicyDocument +from cloudsplaining.scan.statement_detail import StatementDetail + +from fix_plugin_aws.resource.base import AwsResource +from fix_plugin_aws.resource.iam import AwsIamUser + +import re +from fix_plugin_aws.access_edges import ( + find_allowed_action, + make_resoruce_regex, + check_statement_match, + check_principal_match, + IamRequestContext, + check_explicit_deny, +) + +from fix_plugin_aws.access_edges_utils import PolicySource, PolicySourceKind + + +def test_find_allowed_action() -> None: + policy_document = { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject"], "Resource": ["arn:aws:s3:::bucket/*"]}, + {"Effect": "Allow", "Action": ["s3:ListBuckets"], "Resource": ["*"]}, + {"Effect": "Deny", "Action": ["s3:DeleteObject"], "Resource": ["arn:aws:s3:::bucket/*"]}, + ], + } + + allowed_actions = find_allowed_action(PolicyDocument(policy_document)) + + assert allowed_actions == {"s3:GetObject", "s3:PutObject", "s3:ListBuckets"} + + +def test_make_resoruce_regex(): + # Test case 1: Wildcard with * + wildcard = "arn:aws:s3:::my-bucket/*" + regex = make_resoruce_regex(wildcard) + assert isinstance(regex, re.Pattern) + assert regex.match("arn:aws:s3:::my-bucket/my-object") + assert not regex.match("arn:aws:s3:::other-bucket/my-object") + + # Test case 2: Wildcard with ? + wildcard = "arn:aws:s3:::my-bucket/?" + regex = make_resoruce_regex(wildcard) + assert isinstance(regex, re.Pattern) + assert regex.match("arn:aws:s3:::my-bucket/a") + assert not regex.match("arn:aws:s3:::my-bucket/ab") + + # Test case 3: Wildcard with multiple * + wildcard = "arn:aws:s3:::*/*" + regex = make_resoruce_regex(wildcard) + assert isinstance(regex, re.Pattern) + assert regex.match("arn:aws:s3:::my-bucket/my-object") + assert regex.match("arn:aws:s3:::other-bucket/another-object") + + # Test case 4: Wildcard with multiple ? + wildcard = "arn:aws:s3:::my-bucket/??" + regex = make_resoruce_regex(wildcard) + assert isinstance(regex, re.Pattern) + assert regex.match("arn:aws:s3:::my-bucket/ab") + assert not regex.match("arn:aws:s3:::my-bucket/abc") + + +def test_check_statement_match1(): + allow_statement = { + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::example-bucket/*", + "Principal": {"AWS": ["arn:aws:iam::123456789012:user/example-user"]}, + } + statement = StatementDetail(allow_statement) + resource = AwsResource(id="bucket", arn="arn:aws:s3:::example-bucket/object.txt") + principal = AwsResource(id="principal", arn="arn:aws:iam::123456789012:user/example-user") + + # Test matching statement + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, principal) + assert result is True + assert constraints == ["arn:aws:s3:::example-bucket/*"] + + # Test wrong effect + result, constraints = check_statement_match(statement, "Deny", "s3:GetObject", resource, principal) + assert result is False + assert constraints == [] + + # wrong principal does not match + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, resource) + assert result is False + + # Test statement with condition + allow_statement["Condition"] = {"StringEquals": {"s3:prefix": "private/"}} + statement = StatementDetail(allow_statement) + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, principal) + assert result is True + + # not providing principaal works + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, principal=None) + assert result is True + + # not providing effect works + result, constraints = check_statement_match( + statement, effect=None, action="s3:GetObject", resource=resource, principal=None + ) + assert result is True + + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, principal) + assert result is True + assert constraints == ["arn:aws:s3:::example-bucket/*"] + + deny_statement = { + "Effect": "Deny", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::example-bucket/*", + "Principal": {"AWS": ["arn:aws:iam::123456789012:user/example-user"]}, + } + + statement = StatementDetail(deny_statement) + result, constraints = check_statement_match(statement, "Deny", "s3:GetObject", resource, principal) + assert result is True + assert constraints == ["arn:aws:s3:::example-bucket/*"] + + # test not resource + not_resource_statement = dict(allow_statement) + del not_resource_statement["Resource"] + not_resource_statement["NotResource"] = "arn:aws:s3:::example-bucket/private/*" + statement = StatementDetail(not_resource_statement) + result, constraints = check_statement_match(statement, "Allow", "s3:GetObject", resource, principal) + assert result is True + assert constraints == ["not arn:aws:s3:::example-bucket/private/*"] + + +def test_check_principal_match() -> None: + principal = AwsIamUser(id="user-id", arn="arn:aws:iam::123456789012:user/user-name") + aws_principal_list = ["*", "arn:aws:iam::123456789012:user/user-name", "user-id", "123456789012"] + + assert check_principal_match(principal, aws_principal_list) is True + + principal = AwsIamUser(id="user-id", arn="arn:aws:iam::123456789012:user/user-name") + aws_principal_list = ["another-arn", "another-id"] + + assert check_principal_match(principal, aws_principal_list) is False + + principal = AwsIamUser(id="user-id", arn="arn:aws:iam::123456789012:user/user-name") + aws_principal_list = ["*"] + + assert check_principal_match(principal, aws_principal_list) is True + + +def test_no_explicit_deny(): + """Test when there is no explicit deny in any policies, expect 'NextStep'.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + identity_policies = [] + permission_boundaries = [] + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket") + action = "s3:GetObject" + resource_based_policies = [] + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + assert result == "NextStep" + + +def test_explicit_deny_in_identity_policy(): + """Test when there is an explicit deny without condition in identity policy, expect 'Denied'.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + assert principal.arn + + policy_json = { + "Version": "2012-10-17", + "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::example-bucket/*"}], + } + policy_document = PolicyDocument(policy_json) + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] + permission_boundaries = [] + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + resource_based_policies = [] + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + assert result == "Denied" + + +def test_explicit_deny_with_condition_in_identity_policy(): + """Test when there is an explicit deny with condition in identity policy, expect list of conditions.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + assert principal.arn + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::example-bucket/*", + "Condition": {"StringNotEquals": {"aws:username": "test-user"}}, + } + ], + } + policy_document = PolicyDocument(policy_json) + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] + permission_boundaries = [] + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + resource_based_policies = [] + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + expected_conditions = [policy_json["Statement"][0]["Condition"]] + assert result == expected_conditions + + +def test_explicit_deny_in_scp(): + """Test when there is an explicit deny without condition in SCP, expect 'Denied'.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + identity_policies = [] + permission_boundaries = [] + + scp_policy_json = { + "Version": "2012-10-17", + "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "*"}], + } + scp_policy_document = PolicyDocument(scp_policy_json) + service_control_policy_levels = [[scp_policy_document]] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + resource_based_policies = [] + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + assert result == "Denied" + + +def test_explicit_deny_with_condition_in_scp(): + """Test when there is an explicit deny with condition in SCP, expect list of conditions.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + identity_policies = [] + permission_boundaries = [] + + scp_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "s3:GetObject", + "Resource": "*", + "Condition": {"Bool": {"aws:SecureTransport": "false"}}, + } + ], + } + scp_policy_document = PolicyDocument(scp_policy_json) + service_control_policy_levels = [ + [ + scp_policy_document, + ] + ] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + resource_based_policies = [] + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + expected_conditions = [scp_policy_json["Statement"][0]["Condition"]] + assert result == expected_conditions + + +def test_explicit_deny_in_resource_policy(): + """Test when there is an explicit deny without condition in resource-based policy, expect 'Denied'.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + identity_policies = [] + permission_boundaries = [] + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::example-bucket/*", + } + ], + } + policy_document = PolicyDocument(policy_json) + resource_based_policies = [ + (PolicySource(kind=PolicySourceKind.Resource, arn="arn:aws:s3:::example-bucket"), policy_document) + ] + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + assert result == "Denied" + + +def test_explicit_deny_with_condition_in_resource_policy(): + """Test when there is an explicit deny with condition in resource-based policy, expect list of conditions.""" + principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") + identity_policies = [] + permission_boundaries = [] + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=principal, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::example-bucket/*", + "Condition": {"IpAddress": {"aws:SourceIp": "192.0.2.0/24"}}, + } + ], + } + policy_document = PolicyDocument(policy_json) + resource_based_policies = [ + (PolicySource(kind=PolicySourceKind.Resource, arn="arn:aws:s3:::example-bucket"), policy_document) + ] + + resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") + action = "s3:GetObject" + + result = check_explicit_deny(request_context, resource, action, resource_based_policies) + expected_conditions = [policy_json["Statement"][0]["Condition"]] + assert result == expected_conditions From 6a273beb33f66cd2fb93213475d5681629165f29 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 20 Sep 2024 15:30:41 +0000 Subject: [PATCH 11/47] linter fix --- plugins/aws/test/acccess_edges_test.py | 95 +++++++++++--------------- 1 file changed, 38 insertions(+), 57 deletions(-) diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 179b4c6cbb..8ee713e133 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -3,6 +3,7 @@ from fix_plugin_aws.resource.base import AwsResource from fix_plugin_aws.resource.iam import AwsIamUser +from typing import Any, Dict, List import re from fix_plugin_aws.access_edges import ( @@ -32,7 +33,7 @@ def test_find_allowed_action() -> None: assert allowed_actions == {"s3:GetObject", "s3:PutObject", "s3:ListBuckets"} -def test_make_resoruce_regex(): +def test_make_resoruce_regex() -> None: # Test case 1: Wildcard with * wildcard = "arn:aws:s3:::my-bucket/*" regex = make_resoruce_regex(wildcard) @@ -62,7 +63,7 @@ def test_make_resoruce_regex(): assert not regex.match("arn:aws:s3:::my-bucket/abc") -def test_check_statement_match1(): +def test_check_statement_match1() -> None: allow_statement = { "Effect": "Allow", "Action": "s3:GetObject", @@ -146,41 +147,37 @@ def test_check_principal_match() -> None: assert check_principal_match(principal, aws_principal_list) is True -def test_no_explicit_deny(): +def test_no_explicit_deny() -> None: """Test when there is no explicit deny in any policies, expect 'NextStep'.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") - identity_policies = [] - permission_boundaries = [] - service_control_policy_levels = [] request_context = IamRequestContext( principal=principal, - identity_policies=identity_policies, - permission_boundaries=permission_boundaries, - service_control_policy_levels=service_control_policy_levels, + identity_policies=[], + permission_boundaries=[], + service_control_policy_levels=[], ) resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket") action = "s3:GetObject" - resource_based_policies = [] - result = check_explicit_deny(request_context, resource, action, resource_based_policies) + result = check_explicit_deny(request_context, resource, action, resource_based_policies=[]) assert result == "NextStep" -def test_explicit_deny_in_identity_policy(): +def test_explicit_deny_in_identity_policy() -> None: """Test when there is an explicit deny without condition in identity policy, expect 'Denied'.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") assert principal.arn - policy_json = { + policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::example-bucket/*"}], } policy_document = PolicyDocument(policy_json) identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] - permission_boundaries = [] - service_control_policy_levels = [] + permission_boundaries: List[PolicyDocument] = [] + service_control_policy_levels: List[List[PolicyDocument]] = [] request_context = IamRequestContext( principal=principal, @@ -191,18 +188,17 @@ def test_explicit_deny_in_identity_policy(): resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") action = "s3:GetObject" - resource_based_policies = [] - result = check_explicit_deny(request_context, resource, action, resource_based_policies) + result = check_explicit_deny(request_context, resource, action, resource_based_policies=[]) assert result == "Denied" -def test_explicit_deny_with_condition_in_identity_policy(): +def test_explicit_deny_with_condition_in_identity_policy() -> None: """Test when there is an explicit deny with condition in identity policy, expect list of conditions.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") assert principal.arn - policy_json = { + policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [ { @@ -215,32 +211,27 @@ def test_explicit_deny_with_condition_in_identity_policy(): } policy_document = PolicyDocument(policy_json) identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] - permission_boundaries = [] - service_control_policy_levels = [] request_context = IamRequestContext( principal=principal, identity_policies=identity_policies, - permission_boundaries=permission_boundaries, - service_control_policy_levels=service_control_policy_levels, + permission_boundaries=[], + service_control_policy_levels=[], ) resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") action = "s3:GetObject" - resource_based_policies = [] - result = check_explicit_deny(request_context, resource, action, resource_based_policies) + result = check_explicit_deny(request_context, resource, action, resource_based_policies=[]) expected_conditions = [policy_json["Statement"][0]["Condition"]] assert result == expected_conditions -def test_explicit_deny_in_scp(): +def test_explicit_deny_in_scp() -> None: """Test when there is an explicit deny without condition in SCP, expect 'Denied'.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") - identity_policies = [] - permission_boundaries = [] - scp_policy_json = { + scp_policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "*"}], } @@ -249,26 +240,23 @@ def test_explicit_deny_in_scp(): request_context = IamRequestContext( principal=principal, - identity_policies=identity_policies, - permission_boundaries=permission_boundaries, + identity_policies=[], + permission_boundaries=[], service_control_policy_levels=service_control_policy_levels, ) resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") action = "s3:GetObject" - resource_based_policies = [] - result = check_explicit_deny(request_context, resource, action, resource_based_policies) + result = check_explicit_deny(request_context, resource, action, resource_based_policies=[]) assert result == "Denied" -def test_explicit_deny_with_condition_in_scp(): +def test_explicit_deny_with_condition_in_scp() -> None: """Test when there is an explicit deny with condition in SCP, expect list of conditions.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") - identity_policies = [] - permission_boundaries = [] - scp_policy_json = { + scp_policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [ { @@ -288,35 +276,31 @@ def test_explicit_deny_with_condition_in_scp(): request_context = IamRequestContext( principal=principal, - identity_policies=identity_policies, - permission_boundaries=permission_boundaries, + identity_policies=[], + permission_boundaries=[], service_control_policy_levels=service_control_policy_levels, ) resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") action = "s3:GetObject" - resource_based_policies = [] - result = check_explicit_deny(request_context, resource, action, resource_based_policies) + result = check_explicit_deny(request_context, resource, action, resource_based_policies=[]) expected_conditions = [scp_policy_json["Statement"][0]["Condition"]] assert result == expected_conditions -def test_explicit_deny_in_resource_policy(): +def test_explicit_deny_in_resource_policy() -> None: """Test when there is an explicit deny without condition in resource-based policy, expect 'Denied'.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") - identity_policies = [] - permission_boundaries = [] - service_control_policy_levels = [] request_context = IamRequestContext( principal=principal, - identity_policies=identity_policies, - permission_boundaries=permission_boundaries, - service_control_policy_levels=service_control_policy_levels, + identity_policies=[], + permission_boundaries=[], + service_control_policy_levels=[], ) - policy_json = { + policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [ { @@ -339,21 +323,18 @@ def test_explicit_deny_in_resource_policy(): assert result == "Denied" -def test_explicit_deny_with_condition_in_resource_policy(): +def test_explicit_deny_with_condition_in_resource_policy() -> None: """Test when there is an explicit deny with condition in resource-based policy, expect list of conditions.""" principal = AwsIamUser(id="AID1234567890", arn="arn:aws:iam::123456789012:user/test-user") - identity_policies = [] - permission_boundaries = [] - service_control_policy_levels = [] request_context = IamRequestContext( principal=principal, - identity_policies=identity_policies, - permission_boundaries=permission_boundaries, - service_control_policy_levels=service_control_policy_levels, + identity_policies=[], + permission_boundaries=[], + service_control_policy_levels=[], ) - policy_json = { + policy_json: Dict[str, Any] = { "Version": "2012-10-17", "Statement": [ { From 95aede17c14b3770c20e48eaa87f56d120395ff4 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 20 Sep 2024 17:25:40 +0000 Subject: [PATCH 12/47] more fixes --- plugins/aws/fix_plugin_aws/access_edges.py | 15 +++++++++------ plugins/aws/fix_plugin_aws/resource/base.py | 5 +++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 70fe87da5f..523377ec64 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -345,7 +345,7 @@ def check_resource_based_policies( # todo: support cross-account access evaluation arn = ARN(resource.arn) - if arn.service == "iam" or arn.service == "kms": # type: ignore + if arn.service_prefix == "iam" or arn.service_prefix == "kms": pass # todo: implement implicit deny here @@ -391,8 +391,10 @@ def check_identity_based_policies( for statement, resource_constraints in collect_matching_statements( policy=policy, effect="Allow", action=action, resource=resource, principal=None ): - assert isinstance(statement.condition, dict) - scopes.append(PermissionScope(source, resource_constraints, [statement.condition])) + conditions = [] + if statement.condition: + conditions.append(statement.condition) + scopes.append(PermissionScope(source, resource_constraints, conditions)) return scopes @@ -552,12 +554,13 @@ class AccessEdgeCreator: def __init__(self, builder: GraphBuilder): self.builder = builder self.principals: List[IamRequestContext] = [] + self._init_principals() - def init_principals(self) -> None: + def _init_principals(self) -> None: for node in self.builder.nodes(clazz=AwsResource): if isinstance(node, AwsIamUser): - identity_based_policies = self.get_identity_based_policies(node) + identity_based_policies = self._get_identity_based_policies(node) permission_boundaries: List[PolicyDocument] = [] # todo: add this # todo: https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html service_control_policy_levels: List[List[PolicyDocument]] = [] @@ -571,7 +574,7 @@ def init_principals(self) -> None: self.principals.append(request_context) - def get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[PolicySource, PolicyDocument]]: + def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[PolicySource, PolicyDocument]]: if isinstance(principal, AwsIamUser): inline_policies = [ ( diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index f71dbd50a2..793dda32f9 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -35,7 +35,7 @@ from fixlib.config import Config, current_config from fixlib.core.actions import CoreFeedback, SuppressWithFeedback from fixlib.graph import ByNodeId, BySearchCriteria, EdgeKey, Graph, NodeSelector -from fixlib.json import from_json, value_in_path +from fixlib.json import from_json, to_json, value_in_path from fixlib.json_bender import Bender, bend from fixlib.lock import RWLock from fixlib.proc import set_thread_name @@ -577,7 +577,8 @@ def add_edge( if isinstance(from_node, AwsResource) and isinstance(to_n, AwsResource): start, end = (to_n, from_node) if reverse else (from_node, to_n) with self.graph_edges_access.write_access: - self.graph.add_edge(start, end, edge_type=edge_type, permissions=permissions) + permissions_json = to_json(permissions) if permissions else None + self.graph.add_edge(start, end, edge_type=edge_type, permissions=permissions_json) def add_deferred_edge( self, from_node: BaseResource, edge_type: EdgeType, to_node: str, reverse: bool = False From 066eb6539d6d9ba0650442f8bc637c4e346291c7 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Sun, 22 Sep 2024 17:58:26 +0000 Subject: [PATCH 13/47] add even more tests --- plugins/aws/fix_plugin_aws/access_edges.py | 14 +- plugins/aws/test/acccess_edges_test.py | 543 ++++++++++++++++++++- 2 files changed, 552 insertions(+), 5 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 523377ec64..764defd3d2 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -431,8 +431,15 @@ def is_service_linked_role(principal: AwsResource) -> bool: def get_action_level(action: str) -> str: service, action_name = action.split(":") + level = "Unknown" action_data = get_action_data(service, action_name) - level: str = [info["access_level"] for info in action_data[service] if action == info["action"]][0] + if not action_data: + return level + if len(action_data[service]) > 0: + for info in action_data[service]: + if action == info["action"]: + level = info["access_level"] + break return level @@ -509,14 +516,13 @@ def check_policies( identity_based_allowed = check_identity_based_policies(request_context, resource, action) if not identity_based_allowed: return None + allowed_scopes.extend(identity_based_allowed) # 6. check for session policies # we don't collect session principals and session policies, so this step is skipped # 7. if we reached here, the action is allowed - service, action_name = action.split(":") - action_data = get_action_data(service, action_name) - level = [info["access_level"] for info in action_data[service] if action == info["action"]][0] + level = get_action_level(action) final_scopes: List[PermissionScope] = [] for scope in allowed_scopes: diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 8ee713e133..54e555bcc7 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -13,9 +13,10 @@ check_principal_match, IamRequestContext, check_explicit_deny, + compute_permissions, ) -from fix_plugin_aws.access_edges_utils import PolicySource, PolicySourceKind +from fix_plugin_aws.access_edges_utils import PolicySource, PolicySourceKind, PermissionScope def test_find_allowed_action() -> None: @@ -357,3 +358,543 @@ def test_explicit_deny_with_condition_in_resource_policy() -> None: result = check_explicit_deny(request_context, resource, action, resource_based_policies) expected_conditions = [policy_json["Statement"][0]["Condition"]] assert result == expected_conditions + + +def test_compute_permissions_user_inline_policy_allow(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3GetObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + assert len(permissions) == 1 + assert permissions[0].action == "s3:ListBucket" + assert permissions[0].level == "List" + assert len(permissions[0].scopes) == 1 + assert permissions[0].scopes[0] == PermissionScope( + PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), + ["arn:aws:s3:::my-test-bucket"], + [], + [], + [], + ) + + +def test_compute_permissions_user_inline_policy_allow_with_conditions(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + condition = {"IpAddress": {"aws:SourceIp": "1.1.1.1"}} + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3GetObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + "Condition": condition, + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + assert len(permissions) == 1 + assert permissions[0].action == "s3:ListBucket" + assert permissions[0].level == "List" + assert len(permissions[0].scopes) == 1 + assert permissions[0].scopes[0] == PermissionScope( + PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), + ["arn:aws:s3:::my-test-bucket"], + [condition], + [], + [], + ) + + +def test_compute_permissions_user_inline_policy_deny(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyS3PutObject", + "Effect": "Deny", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::my-test-bucket/*", + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 0 + + +def test_compute_permissions_user_inline_policy_deny_with_condition(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + condition = {"IpAddress": {"aws:SourceIp": "1.1.1.1"}} + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyS3PutObject", + "Effect": "Deny", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::my-test-bucket/*", + "Condition": condition, + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + # deny does not grant any permissions by itself, even if the condition is met + assert len(permissions) == 0 + + +def test_deny_overrides_allow(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + deny_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyS3PutObject", + "Effect": "Deny", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + deny_policy_document = PolicyDocument(deny_policy_json) + + allow_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3PutObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + allow_policy_document = PolicyDocument(allow_policy_json) + + identity_policies = [ + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + ] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 0 + + +def test_deny_different_action_does_not_override_allow(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + deny_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyS3PutObject", + "Effect": "Deny", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + deny_policy_document = PolicyDocument(deny_policy_json) + + allow_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3PutObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + allow_policy_document = PolicyDocument(allow_policy_json) + + identity_policies = [ + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + ] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 1 + + +def test_deny_overrides_allow_with_condition(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + condition = {"IpAddress": {"aws:SourceIp": "1.1.1.1"}} + + deny_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyS3PutObject", + "Effect": "Deny", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + "Condition": condition, + } + ], + } + deny_policy_document = PolicyDocument(deny_policy_json) + + allow_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3PutObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + allow_policy_document = PolicyDocument(allow_policy_json) + + identity_policies = [ + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + ] + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 1 + p = permissions[0] + assert p.action == "s3:ListBucket" + assert p.level == "List" + assert len(p.scopes) == 1 + s = p.scopes[0] + assert s.source.kind == PolicySourceKind.Principal + assert s.source.arn == user.arn + assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + assert s.deny_conditions == [condition] + + +def test_compute_permissions_resource_based_policy_allow(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::111122223333:user/test-user") + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + assert bucket.arn + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowCrossAccountAccess", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::111122223333:user/test-user"}, + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [] + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [(PolicySource(kind=PolicySourceKind.Resource, arn=bucket.arn), policy_document)] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 1 + p = permissions[0] + assert p.action == "s3:ListBucket" + assert p.level == "List" + assert len(p.scopes) == 1 + s = p.scopes[0] + assert s.source.kind == PolicySourceKind.Resource + assert s.source.arn == bucket.arn + assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + + +def test_compute_permissions_permission_boundary_restrict(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + identity_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3DeleteObject", + "Effect": "Allow", + "Action": "s3:DeleteBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + }, + { + "Sid": "AllowS3GetObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + }, + ], + } + identity_policy_document = PolicyDocument(identity_policy_json) + + permission_boundary_json = { + "Version": "2012-10-17", + "Statement": [ + {"Sid": "Boundary", "Effect": "Allow", "Action": ["s3:ListBucket", "s3:PutObject"], "Resource": "*"} + ], + } + permission_boundary_document = PolicyDocument(permission_boundary_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), identity_policy_document)] + + permission_boundaries = [permission_boundary_document] + + request_context = IamRequestContext( + principal=user, + identity_policies=identity_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=[], + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 1 + p = permissions[0] + assert p.action == "s3:ListBucket" + assert p.level == "List" + assert len(p.scopes) == 1 + s = p.scopes[0] + assert s.source.kind == PolicySourceKind.Principal + assert s.source.arn == user.arn + assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + + +def test_compute_permissions_scp_deny(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + assert user.arn + + ec2_instance = AwsResource(id="instance123", arn="arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0") + + identity_policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowTerminateInstances", + "Effect": "Allow", + "Action": "ec2:TerminateInstances", + "Resource": ec2_instance.arn, + } + ], + } + identity_policy_document = PolicyDocument(identity_policy_json) + + scp_policy_json = { + "Version": "2012-10-17", + "Statement": [ + {"Sid": "DenyTerminateInstances", "Effect": "Deny", "Action": "ec2:TerminateInstances", "Resource": "*"} + ], + } + scp_policy_document = PolicyDocument(scp_policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), identity_policy_document)] + + service_control_policy_levels = [[scp_policy_document]] + + request_context = IamRequestContext( + principal=user, + identity_policies=identity_policies, + permission_boundaries=[], + service_control_policy_levels=service_control_policy_levels, + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=ec2_instance, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 0 + + +def test_compute_permissions_user_with_group_policies(): + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + group = AwsResource(id="group123", arn="arn:aws:iam::123456789012:group/test-group") + assert group.arn + + group_policy_json = { + "Version": "2012-10-17", + "Statement": [ + {"Sid": "AllowS3ListBucket", "Effect": "Allow", "Action": "s3:ListBucket", "Resource": bucket.arn} + ], + } + group_policy_document = PolicyDocument(group_policy_json) + + identity_policies = [] + + identity_policies.append((PolicySource(kind=PolicySourceKind.Group, arn=group.arn), group_policy_document)) + + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + resource_based_policies = [] + + permissions = compute_permissions( + resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + assert len(permissions) == 1 + p = permissions[0] + assert p.action == "s3:ListBucket" + assert p.level == "List" + assert len(p.scopes) == 1 + s = p.scopes[0] + assert s.source.kind == PolicySourceKind.Group + assert s.source.arn == group.arn + assert s.constraints == [bucket.arn] + + +def test_compute_permissions_implicit_deny(): + # Create a user + user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") + + # Create a resource (DynamoDB table) + table = AwsResource(id="table123", arn="arn:aws:dynamodb:us-east-1:123456789012:table/my-table") + + # No identity policies + identity_policies = [] + + # Create the request context + request_context = IamRequestContext( + principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + # No resource-based policies + resource_based_policies = [] + + # Compute permissions + permissions = compute_permissions( + resource=table, iam_context=request_context, resource_based_policies=resource_based_policies + ) + + # Assert that permissions do not include any actions (implicit deny) + assert len(permissions) == 0 From 13ac740dfcaa170a801b9e3af2d299ad42ccd7f9 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 23 Sep 2024 09:00:27 +0000 Subject: [PATCH 14/47] linter fixes --- plugins/aws/test/acccess_edges_test.py | 104 ++++++------------------- 1 file changed, 25 insertions(+), 79 deletions(-) diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 54e555bcc7..70af817285 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -360,7 +360,7 @@ def test_explicit_deny_with_condition_in_resource_policy() -> None: assert result == expected_conditions -def test_compute_permissions_user_inline_policy_allow(): +def test_compute_permissions_user_inline_policy_allow() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -385,11 +385,7 @@ def test_compute_permissions_user_inline_policy_allow(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" assert permissions[0].level == "List" @@ -403,7 +399,7 @@ def test_compute_permissions_user_inline_policy_allow(): ) -def test_compute_permissions_user_inline_policy_allow_with_conditions(): +def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -431,11 +427,7 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" assert permissions[0].level == "List" @@ -449,7 +441,7 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions(): ) -def test_compute_permissions_user_inline_policy_deny(): +def test_compute_permissions_user_inline_policy_deny() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -474,16 +466,12 @@ def test_compute_permissions_user_inline_policy_deny(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 0 -def test_compute_permissions_user_inline_policy_deny_with_condition(): +def test_compute_permissions_user_inline_policy_deny_with_condition() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -511,17 +499,13 @@ def test_compute_permissions_user_inline_policy_deny_with_condition(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) # deny does not grant any permissions by itself, even if the condition is met assert len(permissions) == 0 -def test_deny_overrides_allow(): +def test_deny_overrides_allow() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -562,16 +546,12 @@ def test_deny_overrides_allow(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 0 -def test_deny_different_action_does_not_override_allow(): +def test_deny_different_action_does_not_override_allow() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -612,16 +592,12 @@ def test_deny_different_action_does_not_override_allow(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 -def test_deny_overrides_allow_with_condition(): +def test_deny_overrides_allow_with_condition() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -665,11 +641,7 @@ def test_deny_overrides_allow_with_condition(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 p = permissions[0] @@ -683,7 +655,7 @@ def test_deny_overrides_allow_with_condition(): assert s.deny_conditions == [condition] -def test_compute_permissions_resource_based_policy_allow(): +def test_compute_permissions_resource_based_policy_allow() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::111122223333:user/test-user") bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") @@ -703,9 +675,8 @@ def test_compute_permissions_resource_based_policy_allow(): } policy_document = PolicyDocument(policy_json) - identity_policies = [] request_context = IamRequestContext( - principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + principal=user, identity_policies=[], permission_boundaries=[], service_control_policy_levels=[] ) resource_based_policies = [(PolicySource(kind=PolicySourceKind.Resource, arn=bucket.arn), policy_document)] @@ -725,7 +696,7 @@ def test_compute_permissions_resource_based_policy_allow(): assert s.constraints == ["arn:aws:s3:::my-test-bucket"] -def test_compute_permissions_permission_boundary_restrict(): +def test_compute_permissions_permission_boundary_restrict() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -769,11 +740,7 @@ def test_compute_permissions_permission_boundary_restrict(): service_control_policy_levels=[], ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 p = permissions[0] @@ -786,7 +753,7 @@ def test_compute_permissions_permission_boundary_restrict(): assert s.constraints == ["arn:aws:s3:::my-test-bucket"] -def test_compute_permissions_scp_deny(): +def test_compute_permissions_scp_deny() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") assert user.arn @@ -824,16 +791,12 @@ def test_compute_permissions_scp_deny(): service_control_policy_levels=service_control_policy_levels, ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=ec2_instance, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=ec2_instance, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 0 -def test_compute_permissions_user_with_group_policies(): +def test_compute_permissions_user_with_group_policies() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") @@ -856,11 +819,7 @@ def test_compute_permissions_user_with_group_policies(): principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [] - - permissions = compute_permissions( - resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 p = permissions[0] @@ -873,28 +832,15 @@ def test_compute_permissions_user_with_group_policies(): assert s.constraints == [bucket.arn] -def test_compute_permissions_implicit_deny(): - # Create a user +def test_compute_permissions_implicit_deny() -> None: user = AwsIamUser(id="user123", arn="arn:aws:iam::123456789012:user/test-user") - - # Create a resource (DynamoDB table) table = AwsResource(id="table123", arn="arn:aws:dynamodb:us-east-1:123456789012:table/my-table") - # No identity policies - identity_policies = [] - - # Create the request context request_context = IamRequestContext( - principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + principal=user, identity_policies=[], permission_boundaries=[], service_control_policy_levels=[] ) - # No resource-based policies - resource_based_policies = [] - - # Compute permissions - permissions = compute_permissions( - resource=table, iam_context=request_context, resource_based_policies=resource_based_policies - ) + permissions = compute_permissions(resource=table, iam_context=request_context, resource_based_policies=[]) # Assert that permissions do not include any actions (implicit deny) assert len(permissions) == 0 From f63658046c0169a8b2c50200d34ec985f9b917fd Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 23 Sep 2024 10:01:32 +0000 Subject: [PATCH 15/47] PR feedback --- fixlib/fixlib/baseresources.py | 7 +- plugins/aws/fix_plugin_aws/access_edges.py | 33 ++++++--- .../aws/fix_plugin_aws/access_edges_utils.py | 32 +++++---- plugins/aws/fix_plugin_aws/resource/base.py | 8 +-- plugins/aws/fix_plugin_aws/resource/s3.py | 4 +- plugins/aws/test/acccess_edges_test.py | 67 +++++++++---------- 6 files changed, 84 insertions(+), 67 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index efd85813ed..0ed75b5cfb 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -6,7 +6,7 @@ from abc import ABC from copy import deepcopy from datetime import datetime, timezone, timedelta -from enum import Enum, unique +from enum import Enum, StrEnum, unique from functools import wraps, cached_property from typing import Dict, Iterator, List, ClassVar, Optional, TypedDict, Any, TypeVar, Type, Callable, Set, Tuple from collections import defaultdict @@ -1599,4 +1599,9 @@ def delete(self, graph: Any) -> bool: return False +class PolicySourceKind(StrEnum): + Principal = "principal" # e.g. IAM user, attached policy + Group = "group" # policy comes from an IAM group + Resource = "resource" # e.g. s3 bucket policy + resolve_types(BaseResource) # noqa diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 764defd3d2..85992b6046 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -5,15 +5,15 @@ from typing import List, Literal, Set, Optional, Tuple, Union, Pattern from fix_plugin_aws.access_edges_utils import ( + PermissionCondition, PolicySource, PermissionScope, AccessPermission, - PolicySourceKind, HasResourcePolicy, ResourceConstraint, ) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser -from fixlib.baseresources import EdgeType +from fixlib.baseresources import EdgeType, PolicySourceKind from fixlib.types import Json from cloudsplaining.scan.policy_document import PolicyDocument @@ -364,10 +364,19 @@ def check_resource_based_policies( for statement, constraints in matching_statements: if statement.condition: scopes.append( - PermissionScope(source=source, constraints=constraints, allow_conditions=[statement.condition]) + PermissionScope( + source=source, + constraints=constraints, + conditions=PermissionCondition(allow=[statement.condition]), + ) ) else: - scopes.append(PermissionScope(source=source, constraints=constraints, allow_conditions=[])) + scopes.append( + PermissionScope( + source=source, + constraints=constraints, + ) + ) # if we found any allow statements, let's check the principal and act accordingly if scopes: @@ -394,7 +403,9 @@ def check_identity_based_policies( conditions = [] if statement.condition: conditions.append(statement.condition) - scopes.append(PermissionScope(source, resource_constraints, conditions)) + scopes.append( + PermissionScope(source, resource_constraints, conditions=PermissionCondition(allow=conditions)) + ) return scopes @@ -526,7 +537,9 @@ def check_policies( final_scopes: List[PermissionScope] = [] for scope in allowed_scopes: - final_scopes.append(scope.with_deny_conditions(deny_conditions)) + if deny_conditions: + scope = scope.with_deny_conditions(deny_conditions) + final_scopes.append(scope) # return the result return AccessPermission( @@ -584,7 +597,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if isinstance(principal, AwsIamUser): inline_policies = [ ( - PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn or ""), + PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), PolicyDocument(policy.policy_document), ) for policy in principal.user_policies @@ -597,7 +610,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if doc := to_node.policy_document_json(): attached_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn or ""), + PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), PolicyDocument(doc), ) ) @@ -609,7 +622,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if policy.policy_document: group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, arn=group.arn or ""), + PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), PolicyDocument(policy.policy_document), ) ) @@ -619,7 +632,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if doc := group_successor.policy_document_json(): group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, arn=group.arn or ""), + PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), PolicyDocument(doc), ) ) diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py index e3b266e1da..fad6d55a1c 100644 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -1,22 +1,16 @@ -from enum import StrEnum from abc import ABC from attrs import frozen, evolve -from typing import List, Tuple, Any +from typing import List, Optional, Tuple, Any from fixlib.types import Json +from fixlib.baseresources import PolicySourceKind ResourceConstraint = str -class PolicySourceKind(StrEnum): - Principal = "principal" # e.g. IAM user, attached policy - Group = "group" # policy comes from an IAM group - Resource = "resource" # e.g. s3 bucket policy - - @frozen class PolicySource: kind: PolicySourceKind - arn: str + uri: str class HasResourcePolicy(ABC): @@ -25,19 +19,29 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: raise NotImplementedError +@frozen +class PermissionCondition: + # if nonempty and any evals to true, access is granted, otherwise implicitly denied + allow: Optional[List[Json]] = None + # if nonempty and any is evals to false, access is implicitly denied + boundary: Optional[List[Json]] = None + # if nonempty and any evals to true, access is explicitly denied + deny: Optional[List[Json]] = None + + @frozen class PermissionScope: source: PolicySource constraints: List[ResourceConstraint] # aka resource constraints - allow_conditions: List[Json] # if nonempty and any evals to true, access is granted, otherwise implicitly denied - boundary_conditions: List[Json] = [] # if nonempty and any is evals to false, access is implicitly denied - deny_conditions: List[Json] = [] # if nonempty and any evals to true, access is explicitly denied + conditions: Optional[PermissionCondition] = None def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": - return evolve(self, deny_conditions=deny_conditions) + c = self.conditions or PermissionCondition() + return evolve(self, conditions=evolve(c, deny=deny_conditions)) def with_boundary_conditions(self, boundary_conditions: List[Json]) -> "PermissionScope": - return evolve(self, boundary_conditions=boundary_conditions) + c = self.conditions or PermissionCondition() + return evolve(self, conditions=evolve(c, boundary=boundary_conditions)) @frozen diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index 5417191853..50222dc489 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -15,7 +15,6 @@ from attrs import define from boto3.exceptions import Boto3Error -from fix_plugin_aws.access_edges_utils import AccessPermission from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.configuration import AwsConfig from fix_plugin_aws.resource.pricing import AwsPricingPrice @@ -35,7 +34,7 @@ from fixlib.config import Config, current_config from fixlib.core.actions import CoreFeedback, SuppressWithFeedback from fixlib.graph import ByNodeId, BySearchCriteria, EdgeKey, Graph, NodeSelector -from fixlib.json import from_json, to_json, value_in_path +from fixlib.json import from_json, value_in_path from fixlib.json_bender import Bender, bend from fixlib.lock import RWLock from fixlib.proc import set_thread_name @@ -570,15 +569,14 @@ def add_edge( from_node: BaseResource, edge_type: EdgeType = EdgeType.default, reverse: bool = False, - permissions: Optional[List[AccessPermission]] = None, + reported: Optional[Json] = None, **to_node: Any, ) -> None: to_n = self.node(**to_node) if isinstance(from_node, AwsResource) and isinstance(to_n, AwsResource): start, end = (to_n, from_node) if reverse else (from_node, to_n) with self.graph_edges_access.write_access: - permissions_json = to_json(permissions) if permissions else None - self.graph.add_edge(start, end, edge_type=edge_type, permissions=permissions_json) + self.graph.add_edge(start, end, edge_type=edge_type, reported=reported) def add_deferred_edge( self, from_node: BaseResource, edge_type: EdgeType, to_node: str, reverse: bool = False diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index 51f7e78b37..965cba9d11 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -7,12 +7,12 @@ from attr import field from attrs import define -from fix_plugin_aws.access_edges_utils import HasResourcePolicy, PolicySource, PolicySourceKind +from fix_plugin_aws.access_edges_utils import HasResourcePolicy, PolicySource from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder, parse_json from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory from fix_plugin_aws.utils import tags_as_dict -from fixlib.baseresources import BaseBucket, MetricName, PhantomBaseResource, ModelReference +from fixlib.baseresources import BaseBucket, MetricName, PhantomBaseResource, ModelReference, PolicySourceKind from fixlib.graph import Graph from fixlib.json import is_empty, sort_json from fixlib.json_bender import Bender, S, bend, Bend, ForallBend diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 70af817285..7e2c2da353 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -16,7 +16,8 @@ compute_permissions, ) -from fix_plugin_aws.access_edges_utils import PolicySource, PolicySourceKind, PermissionScope +from fix_plugin_aws.access_edges_utils import PermissionCondition, PolicySource, PermissionScope +from fixlib.baseresources import PolicySourceKind def test_find_allowed_action() -> None: @@ -176,7 +177,7 @@ def test_explicit_deny_in_identity_policy() -> None: "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::example-bucket/*"}], } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn), policy_document)] permission_boundaries: List[PolicyDocument] = [] service_control_policy_levels: List[List[PolicyDocument]] = [] @@ -211,7 +212,7 @@ def test_explicit_deny_with_condition_in_identity_policy() -> None: ], } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=principal.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn), policy_document)] request_context = IamRequestContext( principal=principal, @@ -314,7 +315,7 @@ def test_explicit_deny_in_resource_policy() -> None: } policy_document = PolicyDocument(policy_json) resource_based_policies = [ - (PolicySource(kind=PolicySourceKind.Resource, arn="arn:aws:s3:::example-bucket"), policy_document) + (PolicySource(kind=PolicySourceKind.Resource, uri="arn:aws:s3:::example-bucket"), policy_document) ] resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") @@ -349,7 +350,7 @@ def test_explicit_deny_with_condition_in_resource_policy() -> None: } policy_document = PolicyDocument(policy_json) resource_based_policies = [ - (PolicySource(kind=PolicySourceKind.Resource, arn="arn:aws:s3:::example-bucket"), policy_document) + (PolicySource(kind=PolicySourceKind.Resource, uri="arn:aws:s3:::example-bucket"), policy_document) ] resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") @@ -379,7 +380,7 @@ def test_compute_permissions_user_inline_policy_allow() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -390,13 +391,10 @@ def test_compute_permissions_user_inline_policy_allow() -> None: assert permissions[0].action == "s3:ListBucket" assert permissions[0].level == "List" assert len(permissions[0].scopes) == 1 - assert permissions[0].scopes[0] == PermissionScope( - PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), - ["arn:aws:s3:::my-test-bucket"], - [], - [], - [], - ) + s = permissions[0].scopes[0] + assert s.source.kind == PolicySourceKind.Principal + assert s.source.uri == user.arn + assert s.constraints == ["arn:aws:s3:::my-test-bucket"] def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: @@ -421,7 +419,7 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -433,11 +431,9 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: assert permissions[0].level == "List" assert len(permissions[0].scopes) == 1 assert permissions[0].scopes[0] == PermissionScope( - PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), + PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), ["arn:aws:s3:::my-test-bucket"], - [condition], - [], - [], + conditions=PermissionCondition(allow=[condition]), ) @@ -460,7 +456,7 @@ def test_compute_permissions_user_inline_policy_deny() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -493,7 +489,7 @@ def test_compute_permissions_user_inline_policy_deny_with_condition() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -538,8 +534,8 @@ def test_deny_overrides_allow() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -584,8 +580,8 @@ def test_deny_different_action_does_not_override_allow() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -633,8 +629,8 @@ def test_deny_overrides_allow_with_condition() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -650,9 +646,10 @@ def test_deny_overrides_allow_with_condition() -> None: assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal - assert s.source.arn == user.arn + assert s.source.uri == user.arn assert s.constraints == ["arn:aws:s3:::my-test-bucket"] - assert s.deny_conditions == [condition] + assert s.conditions + assert s.conditions.deny == [condition] def test_compute_permissions_resource_based_policy_allow() -> None: @@ -679,7 +676,7 @@ def test_compute_permissions_resource_based_policy_allow() -> None: principal=user, identity_policies=[], permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [(PolicySource(kind=PolicySourceKind.Resource, arn=bucket.arn), policy_document)] + resource_based_policies = [(PolicySource(kind=PolicySourceKind.Resource, uri=bucket.arn), policy_document)] permissions = compute_permissions( resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies @@ -692,7 +689,7 @@ def test_compute_permissions_resource_based_policy_allow() -> None: assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Resource - assert s.source.arn == bucket.arn + assert s.source.uri == bucket.arn assert s.constraints == ["arn:aws:s3:::my-test-bucket"] @@ -729,7 +726,7 @@ def test_compute_permissions_permission_boundary_restrict() -> None: } permission_boundary_document = PolicyDocument(permission_boundary_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), identity_policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), identity_policy_document)] permission_boundaries = [permission_boundary_document] @@ -749,7 +746,7 @@ def test_compute_permissions_permission_boundary_restrict() -> None: assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal - assert s.source.arn == user.arn + assert s.source.uri == user.arn assert s.constraints == ["arn:aws:s3:::my-test-bucket"] @@ -780,7 +777,7 @@ def test_compute_permissions_scp_deny() -> None: } scp_policy_document = PolicyDocument(scp_policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, arn=user.arn), identity_policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), identity_policy_document)] service_control_policy_levels = [[scp_policy_document]] @@ -813,7 +810,7 @@ def test_compute_permissions_user_with_group_policies() -> None: identity_policies = [] - identity_policies.append((PolicySource(kind=PolicySourceKind.Group, arn=group.arn), group_policy_document)) + identity_policies.append((PolicySource(kind=PolicySourceKind.Group, uri=group.arn), group_policy_document)) request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -828,7 +825,7 @@ def test_compute_permissions_user_with_group_policies() -> None: assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Group - assert s.source.arn == group.arn + assert s.source.uri == group.arn assert s.constraints == [bucket.arn] From a7b1af574277928b75ba1f7c9e620d5438bf6c94 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 23 Sep 2024 10:17:22 +0000 Subject: [PATCH 16/47] another linter fix --- fixlib/fixlib/baseresources.py | 1 + fixlib/tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 0ed75b5cfb..b8cc1d2dc4 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1604,4 +1604,5 @@ class PolicySourceKind(StrEnum): Group = "group" # policy comes from an IAM group Resource = "resource" # e.g. s3 bucket policy + resolve_types(BaseResource) # noqa diff --git a/fixlib/tox.ini b/fixlib/tox.ini index 15c37aa06a..008bc3c7b1 100644 --- a/fixlib/tox.ini +++ b/fixlib/tox.ini @@ -25,7 +25,7 @@ commands = flake8 --verbose commands= pytest [testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . +commands = black --line-length 120 --check --diff --target-version py311 . [testenv:mypy] -commands= mypy --install-types --non-interactive --python-version 3.9 --strict fixlib +commands= mypy --install-types --non-interactive --python-version 3.11 --strict fixlib From 5bca38a17478e9f9fe6b8882c7543c3722edb7b4 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 23 Sep 2024 17:53:04 +0000 Subject: [PATCH 17/47] performance improvements --- plugins/aws/fix_plugin_aws/access_edges.py | 89 ++++++++++++------- .../aws/fix_plugin_aws/access_edges_utils.py | 25 ++++-- plugins/aws/fix_plugin_aws/collector.py | 1 + plugins/aws/test/__init__.py | 4 +- plugins/aws/test/acccess_edges_test.py | 29 +++--- 5 files changed, 93 insertions(+), 55 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 85992b6046..fd6f7e442c 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -1,5 +1,5 @@ from functools import lru_cache -from attr import define, frozen +from attr import frozen, define from fix_plugin_aws.resource.base import AwsResource, GraphBuilder from typing import List, Literal, Set, Optional, Tuple, Union, Pattern @@ -14,23 +14,24 @@ ) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser from fixlib.baseresources import EdgeType, PolicySourceKind +from fixlib.json import to_json, to_json_str from fixlib.types import Json from cloudsplaining.scan.policy_document import PolicyDocument from cloudsplaining.scan.statement_detail import StatementDetail -from policy_sentry.querying.actions import get_action_data +from policy_sentry.querying.actions import get_action_data, get_actions_matching_arn from policy_sentry.querying.all import get_all_actions -from policy_sentry.util.arns import ARN +from policy_sentry.util.arns import ARN, get_service_from_arn import re import logging +log = logging.getLogger("fix.plugins.aws") -log = logging.getLogger(__name__) ALL_ACTIONS = get_all_actions() -@define +@define(slots=True) class IamRequestContext: principal: AwsResource identity_policies: List[Tuple[PolicySource, PolicyDocument]] @@ -54,29 +55,39 @@ def all_policies( IamAction = str -@lru_cache(maxsize=1024) -def find_allowed_action(policy_document: PolicyDocument) -> Set[IamAction]: +def find_allowed_action(policy_document: PolicyDocument, service_prefix: str) -> Set[IamAction]: allowed_actions: Set[IamAction] = set() for statement in policy_document.statements: if statement.effect_allow: - allowed_actions.update(get_expanded_action(statement)) + allowed_actions.update(get_expanded_action(statement, service_prefix)) return allowed_actions -def find_all_allowed_actions(all_involved_policies: List[PolicyDocument]) -> Set[IamAction]: - allowed_actions: Set[IamAction] = set() +def find_all_allowed_actions(all_involved_policies: List[PolicyDocument], resource_arn: str) -> Set[IamAction]: + resource_actions = set() + try: + resource_actions = set(get_actions_matching_arn(resource_arn)) + except Exception as e: + log.warning(f"Error when trying to get actions matching ARN {resource_arn}: {e}") + + service_prefix = "" + try: + service_prefix = get_service_from_arn(resource_arn) + except Exception as e: + log.warning(f"Error when trying to get service prefix from ARN {resource_arn}: {e}") + policy_actions: Set[IamAction] = set() for p in all_involved_policies: - allowed_actions.update(find_allowed_action(p)) - return allowed_actions + policy_actions.update(find_allowed_action(p, service_prefix)) + return policy_actions.intersection(resource_actions) -@lru_cache(maxsize=1024) -def get_expanded_action(statement: StatementDetail) -> Set[str]: +def get_expanded_action(statement: StatementDetail, service_prefix: str) -> Set[str]: actions = set() expanded: List[str] = statement.expanded_actions or [] for action in expanded: - actions.add(action) + if action.startswith(f"{service_prefix}:"): + actions.add(action) return actions @@ -366,15 +377,15 @@ def check_resource_based_policies( scopes.append( PermissionScope( source=source, - constraints=constraints, - conditions=PermissionCondition(allow=[statement.condition]), + constraints=tuple(constraints), + conditions=PermissionCondition(allow=(to_json_str(statement.condition),)), ) ) else: scopes.append( PermissionScope( source=source, - constraints=constraints, + constraints=tuple(constraints), ) ) @@ -400,12 +411,11 @@ def check_identity_based_policies( for statement, resource_constraints in collect_matching_statements( policy=policy, effect="Allow", action=action, resource=resource, principal=None ): - conditions = [] + conditions = None if statement.condition: - conditions.append(statement.condition) - scopes.append( - PermissionScope(source, resource_constraints, conditions=PermissionCondition(allow=conditions)) - ) + conditions = PermissionCondition(allow=(to_json_str(statement.condition),)) + + scopes.append(PermissionScope(source, tuple(resource_constraints), conditions=conditions)) return scopes @@ -498,11 +508,11 @@ def check_policies( ) if isinstance(resource_result, FinalAllow): scopes = resource_result.scopes - final_resource_scopes: List[PermissionScope] = [] + final_resource_scopes: Set[PermissionScope] = set() for scope in scopes: - final_resource_scopes.append(scope.with_deny_conditions(deny_conditions)) + final_resource_scopes.add(scope.with_deny_conditions(deny_conditions)) - return AccessPermission(action=action, level=get_action_level(action), scopes=final_resource_scopes) + return AccessPermission(action=action, level=get_action_level(action), scopes=tuple(final_resource_scopes)) if isinstance(resource_result, Continue): scopes = resource_result.scopes allowed_scopes.extend(scopes) @@ -535,17 +545,23 @@ def check_policies( # 7. if we reached here, the action is allowed level = get_action_level(action) - final_scopes: List[PermissionScope] = [] + final_scopes: Set[PermissionScope] = set() for scope in allowed_scopes: if deny_conditions: scope = scope.with_deny_conditions(deny_conditions) - final_scopes.append(scope) + final_scopes.add(scope) + + # if there is a scope with no conditions, we can ignore everything else + for scope in final_scopes: + if scope.has_no_condititons(): + final_scopes = {scope} + break # return the result return AccessPermission( action=action, level=level, - scopes=final_scopes, + scopes=tuple(final_scopes), ) @@ -555,8 +571,9 @@ def compute_permissions( resource_based_policies: List[Tuple[PolicySource, PolicyDocument]], ) -> List[AccessPermission]: + assert resource.arn # step 1: find the relevant action to check - relevant_actions = find_all_allowed_actions(iam_context.all_policies(resource_based_policies)) + relevant_actions = find_all_allowed_actions(iam_context.all_policies(resource_based_policies), resource.arn) all_permissions: List[AccessPermission] = [] @@ -610,7 +627,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if doc := to_node.policy_document_json(): attached_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), + PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), PolicyDocument(doc), ) ) @@ -632,7 +649,7 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol if doc := group_successor.policy_document_json(): group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), + PolicySource(kind=PolicySourceKind.Group, uri=group_successor.arn or ""), PolicyDocument(doc), ) ) @@ -653,6 +670,12 @@ def add_access_edges(self) -> None: permissions = compute_permissions(node, context, resource_policies) + if not permissions: + continue + + json_permissions = to_json(permissions) + reported = {"permissions": json_permissions} + self.builder.add_edge( - from_node=context.principal, edge_type=EdgeType.access, permissions=permissions, to_node=node + from_node=context.principal, edge_type=EdgeType.access, reported=reported, to_node=node ) diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py index fad6d55a1c..baa5531eb1 100644 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -1,11 +1,13 @@ from abc import ABC from attrs import frozen, evolve from typing import List, Optional, Tuple, Any +from fixlib.json import to_json_str from fixlib.types import Json from fixlib.baseresources import PolicySourceKind ResourceConstraint = str +ConditionString = str @frozen class PolicySource: @@ -22,30 +24,39 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: @frozen class PermissionCondition: # if nonempty and any evals to true, access is granted, otherwise implicitly denied - allow: Optional[List[Json]] = None + allow: Optional[Tuple[ConditionString, ...]] = None # if nonempty and any is evals to false, access is implicitly denied - boundary: Optional[List[Json]] = None + boundary: Optional[Tuple[ConditionString, ...]] = None # if nonempty and any evals to true, access is explicitly denied - deny: Optional[List[Json]] = None + deny: Optional[Tuple[ConditionString, ...]] = None @frozen class PermissionScope: source: PolicySource - constraints: List[ResourceConstraint] # aka resource constraints + constraints: Tuple[ResourceConstraint, ...] # aka resource constraints conditions: Optional[PermissionCondition] = None def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": c = self.conditions or PermissionCondition() - return evolve(self, conditions=evolve(c, deny=deny_conditions)) + return evolve(self, conditions=evolve(c, deny=tuple([to_json_str(c) for c in deny_conditions]))) def with_boundary_conditions(self, boundary_conditions: List[Json]) -> "PermissionScope": c = self.conditions or PermissionCondition() - return evolve(self, conditions=evolve(c, boundary=boundary_conditions)) + return evolve(self, conditions=evolve(c, boundary=tuple([to_json_str(c) for c in boundary_conditions]))) + + def has_no_condititons(self) -> bool: + if self.conditions is None: + return True + + if self.conditions.allow is None and self.conditions.boundary is None and self.conditions.deny is None: + return True + + return False @frozen class AccessPermission: action: str level: str - scopes: List[PermissionScope] + scopes: Tuple[PermissionScope, ...] diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index 5355f44a87..1d1a2a7290 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -256,6 +256,7 @@ def get_last_run() -> Optional[datetime]: raise Exception("Only AWS resources expected") # add access edges + log.info(f"[Aws:{self.account.id}] Create access edges.") access_edge_creator = AccessEdgeCreator(global_builder) access_edge_creator.add_access_edges() diff --git a/plugins/aws/test/__init__.py b/plugins/aws/test/__init__.py index 00018ec2e8..4b96212e7b 100644 --- a/plugins/aws/test/__init__.py +++ b/plugins/aws/test/__init__.py @@ -35,7 +35,7 @@ def builder(aws_client: AwsClient, no_feedback: CoreFeedback) -> Iterator[GraphB yield GraphBuilder( Graph(), Cloud(id="aws"), - AwsAccount(id="test"), + AwsAccount(id="test", arn="arn:aws:organizations::123456789012:account/o-exampleorgid/123456789012"), regions[0], {r.id: r for r in regions}, aws_client, @@ -51,5 +51,5 @@ def no_feedback() -> CoreFeedback: @fixture def account_collector(aws_config: AwsConfig, no_feedback: CoreFeedback) -> AwsAccountCollector: - account = AwsAccount(id="test") + account = AwsAccount(id="test", arn="arn:aws:organizations::123456789012:account/o-exampleorgid/123456789012") return AwsAccountCollector(aws_config, Cloud(id="aws"), account, ["us-east-1"], no_feedback, {}) diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 7e2c2da353..55964889ce 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -16,8 +16,9 @@ compute_permissions, ) -from fix_plugin_aws.access_edges_utils import PermissionCondition, PolicySource, PermissionScope +from fix_plugin_aws.access_edges_utils import PolicySource from fixlib.baseresources import PolicySourceKind +from fixlib.json import to_json_str def test_find_allowed_action() -> None: @@ -26,11 +27,12 @@ def test_find_allowed_action() -> None: "Statement": [ {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject"], "Resource": ["arn:aws:s3:::bucket/*"]}, {"Effect": "Allow", "Action": ["s3:ListBuckets"], "Resource": ["*"]}, + {"Effect": "Allow", "Action": ["ec2:DescribeInstances"], "Resource": ["*"]}, {"Effect": "Deny", "Action": ["s3:DeleteObject"], "Resource": ["arn:aws:s3:::bucket/*"]}, ], } - allowed_actions = find_allowed_action(PolicyDocument(policy_document)) + allowed_actions = find_allowed_action(PolicyDocument(policy_document), "s3") assert allowed_actions == {"s3:GetObject", "s3:PutObject", "s3:ListBuckets"} @@ -394,7 +396,7 @@ def test_compute_permissions_user_inline_policy_allow() -> None: s = permissions[0].scopes[0] assert s.source.kind == PolicySourceKind.Principal assert s.source.uri == user.arn - assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: @@ -430,11 +432,12 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: assert permissions[0].action == "s3:ListBucket" assert permissions[0].level == "List" assert len(permissions[0].scopes) == 1 - assert permissions[0].scopes[0] == PermissionScope( - PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), - ["arn:aws:s3:::my-test-bucket"], - conditions=PermissionCondition(allow=[condition]), - ) + s = permissions[0].scopes[0] + assert s.source.kind == PolicySourceKind.Principal + assert s.source.uri == user.arn + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) + assert s.conditions + assert s.conditions.allow == (to_json_str(condition),) def test_compute_permissions_user_inline_policy_deny() -> None: @@ -647,9 +650,9 @@ def test_deny_overrides_allow_with_condition() -> None: s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal assert s.source.uri == user.arn - assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) assert s.conditions - assert s.conditions.deny == [condition] + assert s.conditions.deny == (to_json_str(condition),) def test_compute_permissions_resource_based_policy_allow() -> None: @@ -690,7 +693,7 @@ def test_compute_permissions_resource_based_policy_allow() -> None: s = p.scopes[0] assert s.source.kind == PolicySourceKind.Resource assert s.source.uri == bucket.arn - assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) def test_compute_permissions_permission_boundary_restrict() -> None: @@ -747,7 +750,7 @@ def test_compute_permissions_permission_boundary_restrict() -> None: s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal assert s.source.uri == user.arn - assert s.constraints == ["arn:aws:s3:::my-test-bucket"] + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) def test_compute_permissions_scp_deny() -> None: @@ -826,7 +829,7 @@ def test_compute_permissions_user_with_group_policies() -> None: s = p.scopes[0] assert s.source.kind == PolicySourceKind.Group assert s.source.uri == group.arn - assert s.constraints == [bucket.arn] + assert s.constraints == (bucket.arn,) def test_compute_permissions_implicit_deny() -> None: From ad3aa3c731ac170ba83d28be0df8a6ab70984e97 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 23 Sep 2024 17:53:42 +0000 Subject: [PATCH 18/47] arangodb devcontainer update --- .devcontainer/Dockerfile | 4 ++-- .devcontainer/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 37054d10a8..c8260393f0 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,8 +15,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends vim wget -RUN wget -qO- https://download.arangodb.com/arangodb310/DEBIAN/Release.key | apt-key add - -RUN echo 'deb https://download.arangodb.com/arangodb310/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list +RUN wget -qO- https://download.arangodb.com/arangodb311/DEBIAN/Release.key | apt-key add - +RUN echo 'deb https://download.arangodb.com/arangodb311/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list RUN apt-get install -y apt-transport-https RUN apt-get update RUN apt-get install -y arangodb3-client diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 7609711bcd..e66a97203f 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -23,7 +23,7 @@ services: # (Adding the "ports" property to this file will not forward from the container.) arangodb: - image: arangodb:3.10.3 + image: arangodb:3.11.11 restart: unless-stopped # Uncomment the lines below in case you want to keep arangodb data in a separate volume # volumes: From b8d31e148b115337ff956d9ab49828ae85c077a9 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 24 Sep 2024 11:00:46 +0000 Subject: [PATCH 19/47] linter fix --- plugins/aws/fix_plugin_aws/access_edges.py | 1 - plugins/aws/fix_plugin_aws/access_edges_utils.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index fd6f7e442c..af815e0a68 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -342,7 +342,6 @@ class Continue: # check if the resource based policies allow the action # as a shortcut we return the first allow statement we find, or a first seen condition. -# todo: collect all allow statements and conditions def check_resource_based_policies( principal: AwsResource, action: str, diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py index baa5531eb1..6809339272 100644 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ b/plugins/aws/fix_plugin_aws/access_edges_utils.py @@ -9,6 +9,7 @@ ConditionString = str + @frozen class PolicySource: kind: PolicySourceKind From d625b71c6c65456c13102fdda513eb7b912911b1 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 25 Sep 2024 09:02:13 +0000 Subject: [PATCH 20/47] feature flag --- plugins/aws/fix_plugin_aws/collector.py | 9 +++++---- plugins/aws/fix_plugin_aws/configuration.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index 1d1a2a7290..16837a6130 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -255,10 +255,11 @@ def get_last_run() -> Optional[datetime]: log.warning(f"Unexpected node type {node} in graph") raise Exception("Only AWS resources expected") - # add access edges - log.info(f"[Aws:{self.account.id}] Create access edges.") - access_edge_creator = AccessEdgeCreator(global_builder) - access_edge_creator.add_access_edges() + if global_builder.config.collect_access_edges: + # add access edges + log.info(f"[Aws:{self.account.id}] Create access edges.") + access_edge_creator = AccessEdgeCreator(global_builder) + access_edge_creator.add_access_edges() # final hook when the graph is complete for node, data in list(self.graph.nodes(data=True)): diff --git a/plugins/aws/fix_plugin_aws/configuration.py b/plugins/aws/fix_plugin_aws/configuration.py index e088c79f5f..6d8f7a544f 100644 --- a/plugins/aws/fix_plugin_aws/configuration.py +++ b/plugins/aws/fix_plugin_aws/configuration.py @@ -268,6 +268,10 @@ class AwsConfig: default=True, metadata={"description": "Collect resource usage metrics via CloudWatch, enabled by default"}, ) + collect_access_edges: Optional[bool] = field( + default=True, + metadata={"description": "Collect IAM access edges, enabled by default"}, + ) @staticmethod def from_json(json: Json) -> "AwsConfig": From 4406f65ce66b1777fc5367b176ac3315a9d9c6b3 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 25 Sep 2024 09:02:53 +0000 Subject: [PATCH 21/47] make feature flag false by default --- plugins/aws/fix_plugin_aws/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/configuration.py b/plugins/aws/fix_plugin_aws/configuration.py index 6d8f7a544f..112d061f09 100644 --- a/plugins/aws/fix_plugin_aws/configuration.py +++ b/plugins/aws/fix_plugin_aws/configuration.py @@ -269,7 +269,7 @@ class AwsConfig: metadata={"description": "Collect resource usage metrics via CloudWatch, enabled by default"}, ) collect_access_edges: Optional[bool] = field( - default=True, + default=False, metadata={"description": "Collect IAM access edges, enabled by default"}, ) From a90f579e3e9fe0adb64a6730ca03d9014a08bf1b Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 25 Sep 2024 09:12:11 +0000 Subject: [PATCH 22/47] style fix --- plugins/aws/fix_plugin_aws/resource/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index 50222dc489..0884caf5a9 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -493,9 +493,8 @@ def nodes( ) -> Iterator[AwsResourceType]: with self.graph_nodes_access.read_access: for n in self.graph: - is_clazz = isinstance(n, clazz) if clazz else True if ( - is_clazz + (isinstance(n, clazz) if clazz else True) and (filter(n) if filter else True) and all(getattr(n, k, None) == v for k, v in node.items()) ): # noqa From 0c598d17dc9850aef771c62f2bbb25014d32c8af Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 25 Sep 2024 11:44:06 +0000 Subject: [PATCH 23/47] more PR feedback --- plugins/aws/fix_plugin_aws/access_edges.py | 34 +++++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index af815e0a68..61dc08639c 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -337,7 +337,12 @@ class Continue: scopes: List[PermissionScope] -ResourceBasedPolicyResult = Union[FinalAllow, Continue] +@frozen +class Deny: + pass + + +ResourceBasedPolicyResult = Union[FinalAllow, Continue, Deny] # check if the resource based policies allow the action @@ -352,12 +357,10 @@ def check_resource_based_policies( scopes: List[PermissionScope] = [] - # todo: support cross-account access evaluation - arn = ARN(resource.arn) + explicit_allow_required = False if arn.service_prefix == "iam" or arn.service_prefix == "kms": - pass - # todo: implement implicit deny here + explicit_allow_required = True for source, policy in resource_based_policies: @@ -395,6 +398,11 @@ def check_resource_based_policies( # and we can return the result immediately return FinalAllow(scopes) + # if we have KMS or IAM service, we want an explicit allow + if explicit_allow_required: + if not scopes: + return Deny() + # in case of other IAM principals, allow on resource based policy is not enough and # we need to check the permission boundaries return Continue(scopes) @@ -445,7 +453,12 @@ def check_permission_boundaries( def is_service_linked_role(principal: AwsResource) -> bool: - # todo: implement this + assert principal.arn + if ":role/" in principal.arn: + arn = ARN(principal.arn) + role_name = arn.resource_path + return role_name.startswith("AWSServiceRoleFor") + return False @@ -516,6 +529,9 @@ def check_policies( scopes = resource_result.scopes allowed_scopes.extend(scopes) + if isinstance(resource_result, Deny): + return None + # 4. to make it a bit simpler, we check the permission boundaries before checking identity based policies if len(request_context.permission_boundaries) > 0: permission_boundary_result = check_permission_boundaries(request_context, resource, action) @@ -596,8 +612,10 @@ def _init_principals(self) -> None: if isinstance(node, AwsIamUser): identity_based_policies = self._get_identity_based_policies(node) - permission_boundaries: List[PolicyDocument] = [] # todo: add this - # todo: https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html + # todo: colect these resources + permission_boundaries: List[PolicyDocument] = [] + + # todo: collect these resources service_control_policy_levels: List[List[PolicyDocument]] = [] request_context = IamRequestContext( From b92b726b31298ccceef3d400308900b426b09157 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 25 Sep 2024 13:52:42 +0000 Subject: [PATCH 24/47] more pr feedback fixes --- fixlib/fixlib/baseresources.py | 64 ++++++++++++++++++- plugins/aws/fix_plugin_aws/access_edges.py | 5 +- .../aws/fix_plugin_aws/access_edges_utils.py | 63 ------------------ plugins/aws/fix_plugin_aws/resource/s3.py | 12 +++- plugins/aws/test/acccess_edges_test.py | 3 +- 5 files changed, 75 insertions(+), 72 deletions(-) delete mode 100644 plugins/aws/fix_plugin_aws/access_edges_utils.py diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 49f8d80832..16a41da65b 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -12,10 +12,10 @@ from collections import defaultdict from attr import resolve_types -from attrs import define, field, Factory +from attrs import define, field, Factory, frozen, evolve from prometheus_client import Counter, Summary -from fixlib.json import from_json as _from_json, to_json as _to_json +from fixlib.json import from_json as _from_json, to_json as _to_json, to_json_str from fixlib.logger import log from fixlib.types import Json from fixlib.utils import make_valid_timestamp, utc_str @@ -1617,4 +1617,64 @@ class PolicySourceKind(StrEnum): Resource = "resource" # e.g. s3 bucket policy + + +ResourceConstraint = str + +ConditionString = str + + +@frozen +class PolicySource: + kind: PolicySourceKind + uri: str + + +class HasResourcePolicy(ABC): + # returns a list of all policies that affects the resource (inline, attached, etc.) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: + raise NotImplementedError + + +@frozen +class PermissionCondition: + # if nonempty and any evals to true, access is granted, otherwise implicitly denied + allow: Optional[Tuple[ConditionString, ...]] = None + # if nonempty and any is evals to false, access is implicitly denied + boundary: Optional[Tuple[ConditionString, ...]] = None + # if nonempty and any evals to true, access is explicitly denied + deny: Optional[Tuple[ConditionString, ...]] = None + + +@frozen +class PermissionScope: + source: PolicySource + constraints: Tuple[ResourceConstraint, ...] # aka resource constraints + conditions: Optional[PermissionCondition] = None + + def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": + c = self.conditions or PermissionCondition() + return evolve(self, conditions=evolve(c, deny=tuple([to_json_str(c) for c in deny_conditions]))) + + def with_boundary_conditions(self, boundary_conditions: List[Json]) -> "PermissionScope": + c = self.conditions or PermissionCondition() + return evolve(self, conditions=evolve(c, boundary=tuple([to_json_str(c) for c in boundary_conditions]))) + + def has_no_condititons(self) -> bool: + if self.conditions is None: + return True + + if self.conditions.allow is None and self.conditions.boundary is None and self.conditions.deny is None: + return True + + return False + + +@frozen +class AccessPermission: + action: str + level: str + scopes: Tuple[PermissionScope, ...] + + resolve_types(BaseResource) # noqa diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 61dc08639c..1dc41918f1 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -4,16 +4,15 @@ from typing import List, Literal, Set, Optional, Tuple, Union, Pattern -from fix_plugin_aws.access_edges_utils import ( +from fixlib.baseresources import ( PermissionCondition, PolicySource, PermissionScope, AccessPermission, - HasResourcePolicy, ResourceConstraint, ) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser -from fixlib.baseresources import EdgeType, PolicySourceKind +from fixlib.baseresources import EdgeType, PolicySourceKind, HasResourcePolicy from fixlib.json import to_json, to_json_str from fixlib.types import Json diff --git a/plugins/aws/fix_plugin_aws/access_edges_utils.py b/plugins/aws/fix_plugin_aws/access_edges_utils.py deleted file mode 100644 index 6809339272..0000000000 --- a/plugins/aws/fix_plugin_aws/access_edges_utils.py +++ /dev/null @@ -1,63 +0,0 @@ -from abc import ABC -from attrs import frozen, evolve -from typing import List, Optional, Tuple, Any -from fixlib.json import to_json_str -from fixlib.types import Json -from fixlib.baseresources import PolicySourceKind - -ResourceConstraint = str - -ConditionString = str - - -@frozen -class PolicySource: - kind: PolicySourceKind - uri: str - - -class HasResourcePolicy(ABC): - # returns a list of all policies that affects the resource (inline, attached, etc.) - def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Json]]: - raise NotImplementedError - - -@frozen -class PermissionCondition: - # if nonempty and any evals to true, access is granted, otherwise implicitly denied - allow: Optional[Tuple[ConditionString, ...]] = None - # if nonempty and any is evals to false, access is implicitly denied - boundary: Optional[Tuple[ConditionString, ...]] = None - # if nonempty and any evals to true, access is explicitly denied - deny: Optional[Tuple[ConditionString, ...]] = None - - -@frozen -class PermissionScope: - source: PolicySource - constraints: Tuple[ResourceConstraint, ...] # aka resource constraints - conditions: Optional[PermissionCondition] = None - - def with_deny_conditions(self, deny_conditions: List[Json]) -> "PermissionScope": - c = self.conditions or PermissionCondition() - return evolve(self, conditions=evolve(c, deny=tuple([to_json_str(c) for c in deny_conditions]))) - - def with_boundary_conditions(self, boundary_conditions: List[Json]) -> "PermissionScope": - c = self.conditions or PermissionCondition() - return evolve(self, conditions=evolve(c, boundary=tuple([to_json_str(c) for c in boundary_conditions]))) - - def has_no_condititons(self) -> bool: - if self.conditions is None: - return True - - if self.conditions.allow is None and self.conditions.boundary is None and self.conditions.deny is None: - return True - - return False - - -@frozen -class AccessPermission: - action: str - level: str - scopes: Tuple[PermissionScope, ...] diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index f5f6a80cae..00f80482e2 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -7,12 +7,20 @@ from attr import field from attrs import define -from fix_plugin_aws.access_edges_utils import HasResourcePolicy, PolicySource + from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder, parse_json from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory from fix_plugin_aws.utils import tags_as_dict -from fixlib.baseresources import BaseBucket, MetricName, PhantomBaseResource, ModelReference, PolicySourceKind +from fixlib.baseresources import ( + BaseBucket, + MetricName, + PhantomBaseResource, + ModelReference, + PolicySourceKind, + PolicySource, + HasResourcePolicy, +) from fixlib.graph import Graph from fixlib.json import is_empty, sort_json from fixlib.json_bender import Bender, S, bend, Bend, ForallBend diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 55964889ce..1d02af4453 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -16,8 +16,7 @@ compute_permissions, ) -from fix_plugin_aws.access_edges_utils import PolicySource -from fixlib.baseresources import PolicySourceKind +from fixlib.baseresources import PolicySourceKind, PolicySource from fixlib.json import to_json_str From 57d79703b5158b47c994d3b0c164ceae0bd25244 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Thu, 26 Sep 2024 09:04:50 +0000 Subject: [PATCH 25/47] linter fix --- fixlib/fixlib/baseresources.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 16a41da65b..d0d5a6bcaa 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1617,8 +1617,6 @@ class PolicySourceKind(StrEnum): Resource = "resource" # e.g. s3 bucket policy - - ResourceConstraint = str ConditionString = str From 45ce83c04ec73e8f4e3f1631699592c44ae94c37 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 08:28:12 +0000 Subject: [PATCH 26/47] logging improvement --- plugins/aws/fix_plugin_aws/access_edges.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 1dc41918f1..89a858381d 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -68,13 +68,13 @@ def find_all_allowed_actions(all_involved_policies: List[PolicyDocument], resour try: resource_actions = set(get_actions_matching_arn(resource_arn)) except Exception as e: - log.warning(f"Error when trying to get actions matching ARN {resource_arn}: {e}") + log.info(f"Error when trying to get actions matching ARN {resource_arn}: {e}") service_prefix = "" try: service_prefix = get_service_from_arn(resource_arn) except Exception as e: - log.warning(f"Error when trying to get service prefix from ARN {resource_arn}: {e}") + log.info(f"Error when trying to get service prefix from ARN {resource_arn}: {e}") policy_actions: Set[IamAction] = set() for p in all_involved_policies: policy_actions.update(find_allowed_action(p, service_prefix)) @@ -571,6 +571,8 @@ def check_policies( final_scopes = {scope} break + log.debug(f"Found access permission, {action} is allowed for {resource} by {request_context.principal}, level: {level}. Scopes: {len(final_scopes)}") + # return the result return AccessPermission( action=action, @@ -689,6 +691,8 @@ def add_access_edges(self) -> None: if not permissions: continue + log.debug(f"Adding access edges for {node}, {len(permissions)} permissions found") + json_permissions = to_json(permissions) reported = {"permissions": json_permissions} From 3fff4bb5a81fabb6f1b53532107c47bd00ab956a Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 08:39:30 +0000 Subject: [PATCH 27/47] linter --- plugins/aws/fix_plugin_aws/access_edges.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 89a858381d..3ef607b791 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -571,7 +571,9 @@ def check_policies( final_scopes = {scope} break - log.debug(f"Found access permission, {action} is allowed for {resource} by {request_context.principal}, level: {level}. Scopes: {len(final_scopes)}") + log.debug( + f"Found access permission, {action} is allowed for {resource} by {request_context.principal}, level: {level}. Scopes: {len(final_scopes)}" + ) # return the result return AccessPermission( From 281d9c0a3a424d7d05af877a3772a24880e189f5 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 08:43:34 +0000 Subject: [PATCH 28/47] fix settings.json --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5cbd3d3f0f..41cd803e60 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,7 +13,7 @@ "python.testing.pytestEnabled": false, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", - "editor.formatOnSave": true, + "editor.formatOnSave": true }, "black-formatter.args": [ "--line-length", @@ -31,4 +31,4 @@ "python.analysis.extraPaths": [ "fixlib" ] -} +} \ No newline at end of file From 8be14bbfc60c77b2ceaa586658e4073371e8695e Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 09:57:46 +0000 Subject: [PATCH 29/47] add edge bugfix --- plugins/aws/fix_plugin_aws/access_edges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 3ef607b791..a41396854f 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -699,5 +699,5 @@ def add_access_edges(self) -> None: reported = {"permissions": json_permissions} self.builder.add_edge( - from_node=context.principal, edge_type=EdgeType.access, reported=reported, to_node=node + from_node=context.principal, edge_type=EdgeType.access, reported=reported, node=node ) From 5dd7c476a9a7c1061e654323cb8696789e261574 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 10:02:25 +0000 Subject: [PATCH 30/47] remove debug logging --- plugins/aws/fix_plugin_aws/access_edges.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index a41396854f..36a6a03403 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -571,10 +571,6 @@ def check_policies( final_scopes = {scope} break - log.debug( - f"Found access permission, {action} is allowed for {resource} by {request_context.principal}, level: {level}. Scopes: {len(final_scopes)}" - ) - # return the result return AccessPermission( action=action, @@ -693,8 +689,6 @@ def add_access_edges(self) -> None: if not permissions: continue - log.debug(f"Adding access edges for {node}, {len(permissions)} permissions found") - json_permissions = to_json(permissions) reported = {"permissions": json_permissions} From f798bce71a4c8eedea805b1d85bb67526badc074 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 10:09:58 +0000 Subject: [PATCH 31/47] do not create self loop edges. --- plugins/aws/fix_plugin_aws/access_edges.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 36a6a03403..dfdb406408 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -571,6 +571,10 @@ def check_policies( final_scopes = {scope} break + log.debug( + f"Found access permission, {action} is allowed for {resource} by {request_context.principal}, level: {level}. Scopes: {len(final_scopes)}" + ) + # return the result return AccessPermission( action=action, @@ -676,7 +680,13 @@ def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[Pol def add_access_edges(self) -> None: + principal_arns = set([p.principal.arn for p in self.principals]) + for node in self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None): + if node.arn in principal_arns: + # do not create cycles + continue + for context in self.principals: resource_policies: List[Tuple[PolicySource, PolicyDocument]] = [] From 7304da8161f0ce48abf5f9e9164962b4c62fbd09 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 10:15:42 +0000 Subject: [PATCH 32/47] rename access -> iam edges --- fixlib/fixlib/baseresources.py | 2 +- plugins/aws/fix_plugin_aws/access_edges.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index d0d5a6bcaa..d7d608a627 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -68,7 +68,7 @@ class ModelReference(TypedDict, total=False): class EdgeType(Enum): default = "default" delete = "delete" - access = "access" + iam = "iam" @staticmethod def from_value(value: Optional[str] = None) -> EdgeType: diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index dfdb406408..73f46ee6cd 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -703,5 +703,5 @@ def add_access_edges(self) -> None: reported = {"permissions": json_permissions} self.builder.add_edge( - from_node=context.principal, edge_type=EdgeType.access, reported=reported, node=node + from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node ) From 0ec88a7c45477c1d2b7fcd5000dc777c4c0b2338 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Sep 2024 15:36:19 +0000 Subject: [PATCH 33/47] export iam reported section --- fixlib/fixlib/graph/__init__.py | 12 ++++++++---- plugins/aws/fix_plugin_aws/access_edges.py | 4 +--- plugins/aws/fix_plugin_aws/resource/base.py | 5 ++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/fixlib/fixlib/graph/__init__.py b/fixlib/fixlib/graph/__init__.py index 0b14b08e77..6b5533bb21 100644 --- a/fixlib/fixlib/graph/__init__.py +++ b/fixlib/fixlib/graph/__init__.py @@ -122,7 +122,7 @@ def merge(self, graph: Graph, skip_deferred_edges: bool = False) -> None: try: self._log_edge_creation = False - self.update(edges=graph.edges, nodes=graph.nodes) + self.update(edges=graph.edges(keys=True, data=True), nodes=graph.nodes) self.deferred_edges.extend(graph.deferred_edges) finally: self._log_edge_creation = True @@ -640,17 +640,21 @@ def export_graph(self) -> None: if not self.found_replace_node: log.warning(f"No nodes of kind {self.graph_merge_kind.kind} found in graph") start_time = time() - for edge in self.graph.edges: + for edge in self.graph.edges(keys=True, data=True): from_node = edge[0] to_node = edge[1] if not isinstance(from_node, BaseResource) or not isinstance(to_node, BaseResource): log.error(f"One of {from_node} and {to_node} is no base resource") continue edge_dict = {"from": from_node.chksum, "to": to_node.chksum} - if len(edge) == 3: - key = edge[2] + if key := edge[2]: if isinstance(key, EdgeKey) and key.edge_type != EdgeType.default: edge_dict["edge_type"] = key.edge_type.value + + if key.edge_type == EdgeType.iam: + data = edge[3] + if reported := data.get("reported"): + edge_dict["reported"] = reported edge_json = json.dumps(edge_dict) + "\n" self.tempfile.write(edge_json.encode()) self.total_lines += 1 diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 73f46ee6cd..07afd16ee9 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -702,6 +702,4 @@ def add_access_edges(self) -> None: json_permissions = to_json(permissions) reported = {"permissions": json_permissions} - self.builder.add_edge( - from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node - ) + self.builder.add_edge(from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node) diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index ddceb063a4..d3ba9f2cc2 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -579,7 +579,10 @@ def add_edge( if isinstance(from_node, AwsResource) and isinstance(to_n, AwsResource): start, end = (to_n, from_node) if reverse else (from_node, to_n) with self.graph_edges_access.write_access: - self.graph.add_edge(start, end, edge_type=edge_type, reported=reported) + kwargs: Dict[str, Any] = {} + if reported: + kwargs["reported"] = reported + self.graph.add_edge(start, end, edge_type=edge_type, **kwargs) def add_deferred_edge( self, from_node: BaseResource, edge_type: EdgeType, to_node: str, reverse: bool = False From 10e0f661ed26e43db785520a319ed47a843ab24f Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 1 Oct 2024 09:16:42 +0000 Subject: [PATCH 34/47] PR feedback --- fixlib/fixlib/baseresources.py | 11 ++++++++- fixlib/fixlib/graph/__init__.py | 13 +++++------ plugins/aws/fix_plugin_aws/access_edges.py | 25 +++++++++++++++------ plugins/aws/fix_plugin_aws/configuration.py | 2 +- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index d7d608a627..677691db00 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1668,10 +1668,19 @@ def has_no_condititons(self) -> bool: return False +class PermissionLevel(StrEnum): + List = "list" + Read = "read" + Tagging = "tagging" + Write = "write" + PermissionManagement = "permission" + Unknown = "unknown" # in case a resource is not in the levels database + + @frozen class AccessPermission: action: str - level: str + level: PermissionLevel scopes: Tuple[PermissionScope, ...] diff --git a/fixlib/fixlib/graph/__init__.py b/fixlib/fixlib/graph/__init__.py index 6b5533bb21..cca6812a5e 100644 --- a/fixlib/fixlib/graph/__init__.py +++ b/fixlib/fixlib/graph/__init__.py @@ -640,21 +640,18 @@ def export_graph(self) -> None: if not self.found_replace_node: log.warning(f"No nodes of kind {self.graph_merge_kind.kind} found in graph") start_time = time() - for edge in self.graph.edges(keys=True, data=True): - from_node = edge[0] - to_node = edge[1] + for from_node, to_node, key, data in self.graph.edges(keys=True, data=True): if not isinstance(from_node, BaseResource) or not isinstance(to_node, BaseResource): log.error(f"One of {from_node} and {to_node} is no base resource") continue edge_dict = {"from": from_node.chksum, "to": to_node.chksum} - if key := edge[2]: + if key: if isinstance(key, EdgeKey) and key.edge_type != EdgeType.default: edge_dict["edge_type"] = key.edge_type.value - if key.edge_type == EdgeType.iam: - data = edge[3] - if reported := data.get("reported"): - edge_dict["reported"] = reported + if reported := data.get("reported"): + edge_dict["reported"] = reported + edge_json = json.dumps(edge_dict) + "\n" self.tempfile.write(edge_json.encode()) self.total_lines += 1 diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 07afd16ee9..bee70fb540 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -12,7 +12,7 @@ ResourceConstraint, ) from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser -from fixlib.baseresources import EdgeType, PolicySourceKind, HasResourcePolicy +from fixlib.baseresources import EdgeType, PolicySourceKind, HasResourcePolicy, PermissionLevel from fixlib.json import to_json, to_json_str from fixlib.types import Json @@ -461,18 +461,30 @@ def is_service_linked_role(principal: AwsResource) -> bool: return False -def get_action_level(action: str) -> str: +def get_action_level(action: str) -> PermissionLevel: service, action_name = action.split(":") - level = "Unknown" + level = "" action_data = get_action_data(service, action_name) if not action_data: - return level + return PermissionLevel.Unknown if len(action_data[service]) > 0: for info in action_data[service]: if action == info["action"]: level = info["access_level"] break - return level + match level: + case "List": + return PermissionLevel.List + case "Read": + return PermissionLevel.Read + case "Tagging": + return PermissionLevel.Tagging + case "Write": + return PermissionLevel.Write + case "Permissions management": + return PermissionLevel.PermissionManagement + case _: + return PermissionLevel.Unknown # logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html @@ -699,7 +711,6 @@ def add_access_edges(self) -> None: if not permissions: continue - json_permissions = to_json(permissions) - reported = {"permissions": json_permissions} + reported = to_json({"permissions": permissions}, strip_nulls=True) self.builder.add_edge(from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node) diff --git a/plugins/aws/fix_plugin_aws/configuration.py b/plugins/aws/fix_plugin_aws/configuration.py index 112d061f09..95ac583dfb 100644 --- a/plugins/aws/fix_plugin_aws/configuration.py +++ b/plugins/aws/fix_plugin_aws/configuration.py @@ -270,7 +270,7 @@ class AwsConfig: ) collect_access_edges: Optional[bool] = field( default=False, - metadata={"description": "Collect IAM access edges, enabled by default"}, + metadata={"description": "Collect IAM access edges, disabled by default"}, ) @staticmethod From 700d118ed7c01bd70b03c47c190cc8d89023003f Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 1 Oct 2024 09:28:51 +0000 Subject: [PATCH 35/47] linters and tests --- plugins/aws/fix_plugin_aws/access_edges.py | 25 +++++++++++----------- plugins/aws/test/acccess_edges_test.py | 14 ++++++------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index bee70fb540..94509cc609 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -472,19 +472,18 @@ def get_action_level(action: str) -> PermissionLevel: if action == info["action"]: level = info["access_level"] break - match level: - case "List": - return PermissionLevel.List - case "Read": - return PermissionLevel.Read - case "Tagging": - return PermissionLevel.Tagging - case "Write": - return PermissionLevel.Write - case "Permissions management": - return PermissionLevel.PermissionManagement - case _: - return PermissionLevel.Unknown + if level == "List": + return PermissionLevel.List + elif level == "Read": + return PermissionLevel.Read + elif level == "Tagging": + return PermissionLevel.Tagging + elif level == "Write": + return PermissionLevel.Write + elif level == "Permissions management": + return PermissionLevel.PermissionManagement + else: + return PermissionLevel.Unknown # logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index 1d02af4453..e5a4ad0266 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -16,7 +16,7 @@ compute_permissions, ) -from fixlib.baseresources import PolicySourceKind, PolicySource +from fixlib.baseresources import PolicySourceKind, PolicySource, PermissionLevel from fixlib.json import to_json_str @@ -390,7 +390,7 @@ def test_compute_permissions_user_inline_policy_allow() -> None: permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == "List" + assert permissions[0].level == PermissionLevel.List assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] assert s.source.kind == PolicySourceKind.Principal @@ -429,7 +429,7 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == "List" + assert permissions[0].level == PermissionLevel.List assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] assert s.source.kind == PolicySourceKind.Principal @@ -644,7 +644,7 @@ def test_deny_overrides_allow_with_condition() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == "List" + assert p.level == PermissionLevel.List assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal @@ -687,7 +687,7 @@ def test_compute_permissions_resource_based_policy_allow() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == "List" + assert p.level == PermissionLevel.List assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Resource @@ -744,7 +744,7 @@ def test_compute_permissions_permission_boundary_restrict() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == "List" + assert p.level == PermissionLevel.List assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Principal @@ -823,7 +823,7 @@ def test_compute_permissions_user_with_group_policies() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == "List" + assert p.level == PermissionLevel.List assert len(p.scopes) == 1 s = p.scopes[0] assert s.source.kind == PolicySourceKind.Group From 39de959d5d5e08ab6a5f44d9f410a22f20b2cbd6 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 2 Oct 2024 12:27:18 +0000 Subject: [PATCH 36/47] collect more principals --- plugins/aws/fix_plugin_aws/access_edges.py | 154 +++++++++++++++------ plugins/aws/test/acccess_edges_test.py | 76 +++++++++- 2 files changed, 190 insertions(+), 40 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 94509cc609..d01a130cfd 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -11,7 +11,7 @@ AccessPermission, ResourceConstraint, ) -from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser +from fix_plugin_aws.resource.iam import AwsIamGroup, AwsIamPolicy, AwsIamUser, AwsIamRole from fixlib.baseresources import EdgeType, PolicySourceKind, HasResourcePolicy, PermissionLevel from fixlib.json import to_json, to_json_str from fixlib.types import Json @@ -625,7 +625,7 @@ def _init_principals(self) -> None: for node in self.builder.nodes(clazz=AwsResource): if isinstance(node, AwsIamUser): - identity_based_policies = self._get_identity_based_policies(node) + identity_based_policies = self._get_user_based_policies(node) # todo: colect these resources permission_boundaries: List[PolicyDocument] = [] @@ -641,53 +641,129 @@ def _init_principals(self) -> None: self.principals.append(request_context) - def _get_identity_based_policies(self, principal: AwsResource) -> List[Tuple[PolicySource, PolicyDocument]]: - if isinstance(principal, AwsIamUser): - inline_policies = [ - ( - PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), - PolicyDocument(policy.policy_document), + if isinstance(node, AwsIamGroup): + identity_based_policies = self._get_group_based_policies(node) + # todo: colect these resources + permission_boundaries = [] + # todo: collect these resources + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=node, + identity_policies=identity_based_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, + ) + + self.principals.append(request_context) + + if isinstance(node, AwsIamRole): + identity_based_policies = self._get_role_based_policies(node) + # todo: colect these resources + permission_boundaries = [] + # todo: collect these resources + service_control_policy_levels = [] + + request_context = IamRequestContext( + principal=node, + identity_policies=identity_based_policies, + permission_boundaries=permission_boundaries, + service_control_policy_levels=service_control_policy_levels, ) - for policy in principal.user_policies - if policy.policy_document - ] - attached_policies = [] - group_policies = [] - for _, to_node in self.builder.graph.edges(principal): - if isinstance(to_node, AwsIamPolicy): - if doc := to_node.policy_document_json(): - attached_policies.append( + + self.principals.append(request_context) + + def _get_user_based_policies(self, principal: AwsIamUser) -> List[Tuple[PolicySource, PolicyDocument]]: + inline_policies = [ + ( + PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), + PolicyDocument(policy.policy_document), + ) + for policy in principal.user_policies + if policy.policy_document + ] + attached_policies = [] + group_policies = [] + for _, to_node in self.builder.graph.edges(principal): + if isinstance(to_node, AwsIamPolicy): + if doc := to_node.policy_document_json(): + attached_policies.append( + ( + PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), + PolicyDocument(doc), + ) + ) + + if isinstance(to_node, AwsIamGroup): + group = to_node + # inline group policies + for policy in group.group_policies: + if policy.policy_document: + group_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), - PolicyDocument(doc), + PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), + PolicyDocument(policy.policy_document), ) ) - - if isinstance(to_node, AwsIamGroup): - group = to_node - # inline group policies - for policy in group.group_policies: - if policy.policy_document: + # attached group policies + for _, group_successor in self.builder.graph.edges(group): + if isinstance(group_successor, AwsIamPolicy): + if doc := group_successor.policy_document_json(): group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), - PolicyDocument(policy.policy_document), + PolicySource(kind=PolicySourceKind.Group, uri=group_successor.arn or ""), + PolicyDocument(doc), ) ) - # attached group policies - for _, group_successor in self.builder.graph.edges(group): - if isinstance(group_successor, AwsIamPolicy): - if doc := group_successor.policy_document_json(): - group_policies.append( - ( - PolicySource(kind=PolicySourceKind.Group, uri=group_successor.arn or ""), - PolicyDocument(doc), - ) - ) - return inline_policies + attached_policies + group_policies + return inline_policies + attached_policies + group_policies + + def _get_group_based_policies(self, principal: AwsIamGroup) -> List[Tuple[PolicySource, PolicyDocument]]: + # not really a principal, but could be useful to have access edges for groups + inline_policies = [ + ( + PolicySource(kind=PolicySourceKind.Group, uri=principal.arn or ""), + PolicyDocument(policy.policy_document), + ) + for policy in principal.group_policies + if policy.policy_document + ] + + attached_policies = [] + for _, to_node in self.builder.graph.edges(principal): + if isinstance(to_node, AwsIamPolicy): + if doc := to_node.policy_document_json(): + attached_policies.append( + ( + PolicySource(kind=PolicySourceKind.Group, uri=to_node.arn or ""), + PolicyDocument(doc), + ) + ) + + return inline_policies + attached_policies + + def _get_role_based_policies(self, principal: AwsIamRole) -> List[Tuple[PolicySource, PolicyDocument]]: + inline_policies = [] + for doc in [p.policy_document for p in principal.role_policies if p.policy_document]: + inline_policies.append( + ( + PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), + PolicyDocument(doc), + ) + ) + + attached_policies = [] + for _, to_node in self.builder.graph.edges(principal): + if isinstance(to_node, AwsIamPolicy): + if policy_doc := to_node.policy_document_json(): + attached_policies.append( + ( + PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), + PolicyDocument(policy_doc), + ) + ) - return [] + return inline_policies + attached_policies def add_access_edges(self) -> None: diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index e5a4ad0266..ac31e3a35e 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -2,7 +2,7 @@ from cloudsplaining.scan.statement_detail import StatementDetail from fix_plugin_aws.resource.base import AwsResource -from fix_plugin_aws.resource.iam import AwsIamUser +from fix_plugin_aws.resource.iam import AwsIamUser, AwsIamGroup, AwsIamRole from typing import Any, Dict, List import re @@ -843,3 +843,77 @@ def test_compute_permissions_implicit_deny() -> None: # Assert that permissions do not include any actions (implicit deny) assert len(permissions) == 0 + + +def test_compute_permissions_group_inline_policy_allow() -> None: + group = AwsIamGroup(id="group123", arn="arn:aws:iam::123456789012:group/test-group") + assert group.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3ListBucket", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Group, uri=group.arn), policy_document)] + + request_context = IamRequestContext( + principal=group, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) + + assert len(permissions) == 1 + assert permissions[0].action == "s3:ListBucket" + assert permissions[0].level == PermissionLevel.List + assert len(permissions[0].scopes) == 1 + s = permissions[0].scopes[0] + assert s.source.kind == PolicySourceKind.Group + assert s.source.uri == group.arn + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) + + +def test_compute_permissions_role_inline_policy_allow() -> None: + role = AwsIamRole(id="role123", arn="arn:aws:iam::123456789012:role/test-role") + assert role.arn + + bucket = AwsResource(id="bucket123", arn="arn:aws:s3:::my-test-bucket") + + policy_json = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowS3PutObject", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::my-test-bucket", + } + ], + } + policy_document = PolicyDocument(policy_json) + + identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=role.arn), policy_document)] + + request_context = IamRequestContext( + principal=role, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] + ) + + permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) + + assert len(permissions) == 1 + assert permissions[0].action == "s3:ListBucket" + assert permissions[0].level == PermissionLevel.List + assert len(permissions[0].scopes) == 1 + s = permissions[0].scopes[0] + assert s.source.kind == PolicySourceKind.Principal + assert s.source.uri == role.arn + assert s.constraints == ("arn:aws:s3:::my-test-bucket",) From fabebcf0a22b5d351aa5ee4f1f065330e50076b7 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Wed, 2 Oct 2024 14:30:25 +0000 Subject: [PATCH 37/47] debug logging --- plugins/aws/fix_plugin_aws/access_edges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index d01a130cfd..3ea0080d5e 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -68,13 +68,13 @@ def find_all_allowed_actions(all_involved_policies: List[PolicyDocument], resour try: resource_actions = set(get_actions_matching_arn(resource_arn)) except Exception as e: - log.info(f"Error when trying to get actions matching ARN {resource_arn}: {e}") + log.debug(f"Error when trying to get actions matching ARN {resource_arn}: {e}") service_prefix = "" try: service_prefix = get_service_from_arn(resource_arn) except Exception as e: - log.info(f"Error when trying to get service prefix from ARN {resource_arn}: {e}") + log.debug(f"Error when trying to get service prefix from ARN {resource_arn}: {e}") policy_actions: Set[IamAction] = set() for p in all_involved_policies: policy_actions.update(find_allowed_action(p, service_prefix)) From ef3415246a7e9d06a383d730c66afbc01db05962 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 4 Oct 2024 10:11:01 +0000 Subject: [PATCH 38/47] do not check every action on AWS readonly policy --- plugins/aws/fix_plugin_aws/access_edges.py | 31 +++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 3ea0080d5e..3ed639f849 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -117,6 +117,7 @@ def check_statement_match( action: str, resource: AwsResource, principal: Optional[AwsResource], + source_arn: Optional[str] = None, ) -> Tuple[bool, List[ResourceConstraint]]: """ check if a statement matches the given effect, action, resource and principal, @@ -167,10 +168,18 @@ def check_statement_match( # step 3: check if the action matches action_match = False if statement.actions: - for a in statement.actions: - if expand_wildcards_and_match(identifier=action, wildcard_string=a): + # shortcuts for known AWS managed policies + if source_arn == "arn:aws:iam::aws:policy/ReadOnlyAccess": + action_level = get_action_level(action) + if action_level in [PermissionLevel.Read or PermissionLevel.List]: action_match = True - break + else: + action_match = False + else: + for a in statement.actions: + if expand_wildcards_and_match(identifier=action, wildcard_string=a): + action_match = True + break else: # not_action action_match = True @@ -236,6 +245,7 @@ def collect_matching_statements( action: str, resource: AwsResource, principal: Optional[AwsResource], + source_arn: Optional[str] = None, ) -> List[Tuple[StatementDetail, List[ResourceConstraint]]]: """ resoruce based policies contain principal field and need to be handled differently @@ -248,7 +258,7 @@ def collect_matching_statements( for statement in policy.statements: matches, maybe_resource_constraint = check_statement_match( - statement, effect=effect, action=action, resource=resource, principal=principal + statement, effect=effect, action=action, resource=resource, principal=principal, source_arn=source_arn ) if matches: results.append((statement, maybe_resource_constraint)) @@ -415,7 +425,7 @@ def check_identity_based_policies( for source, policy in request_context.identity_policies: for statement, resource_constraints in collect_matching_statements( - policy=policy, effect="Allow", action=action, resource=resource, principal=None + policy=policy, effect="Allow", action=action, resource=resource, principal=None, source_arn=source.uri ): conditions = None if statement.condition: @@ -769,7 +779,14 @@ def add_access_edges(self) -> None: principal_arns = set([p.principal.arn for p in self.principals]) - for node in self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None): + total_nodes = self.builder.graph.number_of_nodes() + + one_percent = total_nodes // 100 + + for idx, node in enumerate(self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None)): + if idx % one_percent == 0: + log.info(f"Computing access edges: {idx} / {total_nodes}, {idx // one_percent}%") + if node.arn in principal_arns: # do not create cycles continue @@ -789,3 +806,5 @@ def add_access_edges(self) -> None: reported = to_json({"permissions": permissions}, strip_nulls=True) self.builder.add_edge(from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node) + + log.info("Computing access edges: completed") From 1432a295c65b7fbf94fb53be36746551da508844 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 4 Oct 2024 10:45:45 +0000 Subject: [PATCH 39/47] remove progress bar --- plugins/aws/fix_plugin_aws/access_edges.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 3ed639f849..6160f3d032 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -779,13 +779,7 @@ def add_access_edges(self) -> None: principal_arns = set([p.principal.arn for p in self.principals]) - total_nodes = self.builder.graph.number_of_nodes() - - one_percent = total_nodes // 100 - - for idx, node in enumerate(self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None)): - if idx % one_percent == 0: - log.info(f"Computing access edges: {idx} / {total_nodes}, {idx // one_percent}%") + for node in self.builder.nodes(clazz=AwsResource, filter=lambda r: r.arn is not None): if node.arn in principal_arns: # do not create cycles @@ -806,5 +800,3 @@ def add_access_edges(self) -> None: reported = to_json({"permissions": permissions}, strip_nulls=True) self.builder.add_edge(from_node=context.principal, edge_type=EdgeType.iam, reported=reported, node=node) - - log.info("Computing access edges: completed") From 638ee686b75f0b5214f7593a68e20ba1e709c0d4 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 7 Oct 2024 14:22:32 +0000 Subject: [PATCH 40/47] fetch permission boundaries for users and roles --- plugins/aws/fix_plugin_aws/access_edges.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 6160f3d032..cff42292f0 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -636,8 +636,12 @@ def _init_principals(self) -> None: if isinstance(node, AwsIamUser): identity_based_policies = self._get_user_based_policies(node) - # todo: colect these resources + permission_boundaries: List[PolicyDocument] = [] + if (pb := node.user_permissions_boundary) and (pb_arn := pb.permissions_boundary_arn): + for pb_policy in self.builder.nodes(clazz=AwsIamPolicy, filter=lambda p: p.arn == pb_arn): + if pdj := pb_policy.policy_document_json(): + permission_boundaries.append(PolicyDocument(pdj)) # todo: collect these resources service_control_policy_levels: List[List[PolicyDocument]] = [] @@ -653,15 +657,13 @@ def _init_principals(self) -> None: if isinstance(node, AwsIamGroup): identity_based_policies = self._get_group_based_policies(node) - # todo: colect these resources - permission_boundaries = [] # todo: collect these resources service_control_policy_levels = [] request_context = IamRequestContext( principal=node, identity_policies=identity_based_policies, - permission_boundaries=permission_boundaries, + permission_boundaries=[], # permission boundaries are not applicable to groups service_control_policy_levels=service_control_policy_levels, ) @@ -671,6 +673,10 @@ def _init_principals(self) -> None: identity_based_policies = self._get_role_based_policies(node) # todo: colect these resources permission_boundaries = [] + if (pb := node.role_permissions_boundary) and (pb_arn := pb.permissions_boundary_arn): + for pb_policy in self.builder.nodes(clazz=AwsIamPolicy, filter=lambda p: p.arn == pb_arn): + if pdj := pb_policy.policy_document_json(): + permission_boundaries.append(PolicyDocument(pdj)) # todo: collect these resources service_control_policy_levels = [] From eedcfc015b004f2743c4cb4e391178ea93330b8a Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 7 Oct 2024 14:37:01 +0000 Subject: [PATCH 41/47] linter fix --- .devcontainer/Dockerfile | 2 +- plugins/aws/fix_plugin_aws/access_edges.py | 2 +- plugins/aws/tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c8260393f0..79b4b97cf6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/devcontainers/python:3.11 +FROM mcr.microsoft.com/devcontainers/python:3.12 ARG USERNAME=vscode diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index cff42292f0..362ec34e0e 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -663,7 +663,7 @@ def _init_principals(self) -> None: request_context = IamRequestContext( principal=node, identity_policies=identity_based_policies, - permission_boundaries=[], # permission boundaries are not applicable to groups + permission_boundaries=[], # permission boundaries are not applicable to groups service_control_policy_levels=service_control_policy_levels, ) diff --git a/plugins/aws/tox.ini b/plugins/aws/tox.ini index eb2b934b13..0f6055cb16 100644 --- a/plugins/aws/tox.ini +++ b/plugins/aws/tox.ini @@ -27,7 +27,7 @@ commands = flake8 commands= pytest [testenv:black] -commands = black --line-length 120 --check --diff --target-version py39 . +commands = black --line-length 120 --check --diff --target-version py312 . [testenv:mypy] commands= python -m mypy --install-types --non-interactive --python-version 3.12 --strict fix_plugin_aws test From 45a1beeb33b748c824f1275f3b9058699bde1d8e Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 11:22:40 +0000 Subject: [PATCH 42/47] collect more resource-based policies --- .vscode/settings.json | 2 +- plugins/aws/fix_plugin_aws/resource/backup.py | 12 ++- .../aws/fix_plugin_aws/resource/dynamodb.py | 16 +++- plugins/aws/fix_plugin_aws/resource/ecr.py | 10 +- plugins/aws/fix_plugin_aws/resource/efs.py | 12 ++- .../aws/fix_plugin_aws/resource/kinesis.py | 4 +- plugins/aws/fix_plugin_aws/resource/kms.py | 11 ++- .../aws/fix_plugin_aws/resource/lambda_.py | 91 ++++--------------- .../fix_plugin_aws/resource/secretsmanager.py | 40 +++++++- plugins/aws/fix_plugin_aws/resource/sns.py | 12 ++- plugins/aws/fix_plugin_aws/resource/sqs.py | 19 +++- plugins/aws/test/collector_test.py | 2 +- .../get-resource-policy__foo.json | 5 + plugins/aws/test/resources/lambda_test.py | 24 +---- 14 files changed, 134 insertions(+), 126 deletions(-) create mode 100644 plugins/aws/test/resources/files/secretsmanager/get-resource-policy__foo.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 41cd803e60..5e11a76027 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,7 +10,7 @@ "fixworker/test" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": false, + "python.testing.pytestEnabled": true, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true diff --git a/plugins/aws/fix_plugin_aws/resource/backup.py b/plugins/aws/fix_plugin_aws/resource/backup.py index 5a1bd24c6d..dbbbcd3e32 100644 --- a/plugins/aws/fix_plugin_aws/resource/backup.py +++ b/plugins/aws/fix_plugin_aws/resource/backup.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import Any, ClassVar, Dict, Optional, List, Type +from typing import Any, ClassVar, Dict, Optional, List, Tuple, Type from json import loads as json_loads from attrs import define, field @@ -15,7 +15,7 @@ from fix_plugin_aws.resource.redshift import AwsRedshiftCluster from fix_plugin_aws.resource.s3 import AwsS3Bucket from fix_plugin_aws.utils import TagsValue -from fixlib.baseresources import ModelReference +from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind from fixlib.graph import Graph from fixlib.json_bender import F, Bender, S, ForallBend, Bend from fixlib.types import Json @@ -304,7 +304,7 @@ def add_tags(backup_plan: AwsBackupPlan) -> None: @define(eq=False, slots=False) -class AwsBackupVault(BackupResourceTaggable, AwsResource): +class AwsBackupVault(BackupResourceTaggable, AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_backup_vault" _kind_display: ClassVar[str] = "AWS Backup Vault" _kind_description: ClassVar[str] = "AWS Backup Vault is a secure storage container for backup data in AWS Backup. It stores and organizes backup copies, providing encryption and access policies to protect backups. Users can create multiple vaults to separate backups by application, environment, or compliance requirements. AWS Backup Vault supports retention policies and lifecycle management for stored backups." # fmt: skip @@ -341,6 +341,12 @@ class AwsBackupVault(BackupResourceTaggable, AwsResource): lock_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time when Backup Vault Lock configuration becomes immutable, meaning it cannot be changed or deleted. If you applied Vault Lock to your vault without specifying a lock date, you can change your Vault Lock settings, or delete Vault Lock from the vault entirely, at any time. This value is in Unix format, Coordinated Universal Time (UTC), and accurate to milliseconds. For example, the value 1516925490.087 represents Friday, January 26, 2018 12:11:30.087 AM."}) # fmt: skip vault_policy: Optional[Json] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.vault_policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, uri=self.arn or ""), self.vault_policy or {})] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index a5547954b8..ac678969ff 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import ClassVar, Dict, List, Optional, Type, Any +from typing import ClassVar, Dict, List, Optional, Tuple, Type, Any from attrs import define, field from json import loads as json_loads @@ -8,7 +8,7 @@ from fix_plugin_aws.resource.kinesis import AwsKinesisStream from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import ModelReference +from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind from fixlib.graph import Graph from fixlib.json_bender import S, Bend, Bender, ForallBend, bend from fixlib.types import Json @@ -356,7 +356,7 @@ class AwsDynamoDbContinuousBackup: @define(eq=False, slots=False) -class AwsDynamoDbTable(DynamoDbTaggable, AwsResource): +class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_dynamodb_table" _kind_display: ClassVar[str] = "AWS DynamoDB Table" _kind_description: ClassVar[str] = "AWS DynamoDB Table is a fully managed NoSQL database service that stores and retrieves data. It supports key-value and document data models, offering automatic scaling and low-latency performance. DynamoDB Tables handle data storage, indexing, and querying, providing consistent read and write throughput. They offer data encryption, backup, and recovery features for secure and reliable data management." # fmt: skip @@ -419,6 +419,11 @@ class AwsDynamoDbTable(DynamoDbTaggable, AwsResource): dynamodb_continuous_backup: Optional[AwsDynamoDbContinuousBackup] = field(default=None) dynamodb_policy: Optional[Json] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.dynamodb_policy: + return [] + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.dynamodb_policy or {})] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ @@ -498,7 +503,7 @@ def called_mutator_apis(cls) -> List[AwsApiSpec]: @define(eq=False, slots=False) -class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource): +class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_dynamodb_global_table" _kind_display: ClassVar[str] = "AWS DynamoDB Global Table" _kind_description: ClassVar[str] = "AWS DynamoDB Global Table is a feature that replicates DynamoDB tables across multiple AWS regions. It provides multi-region read and write access to data, ensuring low-latency access for globally distributed applications. Global Table maintains consistency across regions, handles conflict resolution, and offers automatic failover, improving availability and disaster recovery capabilities for applications with global user bases." # fmt: skip @@ -525,6 +530,9 @@ class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource): dynamodb_global_table_status: Optional[str] = field(default=None) dynamodb_policy: Optional[Json] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.dynamodb_policy or {})] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/fix_plugin_aws/resource/ecr.py b/plugins/aws/fix_plugin_aws/resource/ecr.py index b8ad1f7a16..7cea09aee6 100644 --- a/plugins/aws/fix_plugin_aws/resource/ecr.py +++ b/plugins/aws/fix_plugin_aws/resource/ecr.py @@ -1,6 +1,6 @@ import json import logging -from typing import ClassVar, Dict, Optional, List, Type, Any +from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any from json import loads as json_loads from attrs import define, field @@ -8,6 +8,7 @@ from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder from fix_plugin_aws.utils import ToDict +from fixlib.baseresources import HasResourcePolicy, PolicySource, PolicySourceKind from fixlib.json import sort_json from fixlib.json_bender import Bender, S, Bend from fixlib.types import Json @@ -25,7 +26,7 @@ class AwsEcrEncryptionConfiguration: @define(eq=False, slots=False) -class AwsEcrRepository(AwsResource): +class AwsEcrRepository(AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_ecr_repository" _kind_display: ClassVar[str] = "AWS ECR Repository" _kind_description: ClassVar[str] = "AWS ECR (Elastic Container Registry) is a managed Docker container registry service. It stores, manages, and deploys container images for applications. ECR integrates with other AWS services, provides secure access control, and supports image scanning for vulnerabilities. Users can push, pull, and share Docker images within their AWS environment or with external parties." # fmt: skip @@ -57,6 +58,11 @@ class AwsEcrRepository(AwsResource): lifecycle_policy: Optional[Json] = field(default=None, metadata={"description": "The repository lifecycle policy."}) # fmt: skip repository_policy: Optional[Json] = field(default=None, metadata={"description": "The repository policy."}) # fmt: skip + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.repository_policy: + return [] + return [(PolicySource(PolicySourceKind.Resource, self.repository_arn or ""), self.repository_policy or {})] + @classmethod def collect_resources(cls, builder: GraphBuilder) -> None: def add_repository_policy(repository: AwsEcrRepository) -> None: diff --git a/plugins/aws/fix_plugin_aws/resource/efs.py b/plugins/aws/fix_plugin_aws/resource/efs.py index 7cd16c17f6..1dced62eda 100644 --- a/plugins/aws/fix_plugin_aws/resource/efs.py +++ b/plugins/aws/fix_plugin_aws/resource/efs.py @@ -1,5 +1,5 @@ import json -from typing import Optional, ClassVar, Dict, List, Type, Any +from typing import Optional, ClassVar, Dict, List, Tuple, Type, Any import math @@ -9,7 +9,7 @@ from fix_plugin_aws.resource.base import AwsApiSpec, GraphBuilder, AwsResource from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import ModelReference, BaseNetworkShare +from fixlib.baseresources import HasResourcePolicy, ModelReference, BaseNetworkShare, PolicySource, PolicySourceKind from fixlib.graph import Graph from fixlib.json import sort_json from fixlib.json_bender import Bender, S, F, Bend @@ -75,7 +75,7 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: @define(eq=False, slots=False) -class AwsEfsFileSystem(EfsTaggable, AwsResource, BaseNetworkShare): +class AwsEfsFileSystem(EfsTaggable, AwsResource, BaseNetworkShare, HasResourcePolicy): kind: ClassVar[str] = "aws_efs_file_system" _kind_display: ClassVar[str] = "AWS EFS File System" _aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/efs/home?region={region}#/file-systems/{FileSystemId}", "arn_tpl": "arn:{partition}:efs:{region}:{account}:file-system/{id}"} # fmt: skip @@ -116,6 +116,12 @@ class AwsEfsFileSystem(EfsTaggable, AwsResource, BaseNetworkShare): availability_zone_name: Optional[str] = field(default=None) file_system_policy: Optional[Json] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.file_system_policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.file_system_policy or {})] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/fix_plugin_aws/resource/kinesis.py b/plugins/aws/fix_plugin_aws/resource/kinesis.py index 9d01c9bdb3..4c600834f8 100644 --- a/plugins/aws/fix_plugin_aws/resource/kinesis.py +++ b/plugins/aws/fix_plugin_aws/resource/kinesis.py @@ -8,7 +8,7 @@ from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import MetricName, ModelReference +from fixlib.baseresources import HasResourcePolicy, MetricName, ModelReference from fixlib.graph import Graph from fixlib.json_bender import Bender, S, Bend, bend, ForallBend from fixlib.types import Json @@ -89,7 +89,7 @@ class AwsKinesisEnhancedMetrics: @define(eq=False, slots=False) -class AwsKinesisStream(AwsResource): +class AwsKinesisStream(AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_kinesis_stream" _kind_display: ClassVar[str] = "AWS Kinesis Stream" _kind_description: ClassVar[str] = "" # fmt: skip diff --git a/plugins/aws/fix_plugin_aws/resource/kms.py b/plugins/aws/fix_plugin_aws/resource/kms.py index 097429dfce..ae019b4dd1 100644 --- a/plugins/aws/fix_plugin_aws/resource/kms.py +++ b/plugins/aws/fix_plugin_aws/resource/kms.py @@ -1,12 +1,12 @@ import json -from typing import ClassVar, Dict, List, Optional, Type, Any +from typing import ClassVar, Dict, List, Optional, Tuple, Type, Any from attrs import define, field from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import BaseAccessKey +from fixlib.baseresources import BaseAccessKey, HasResourcePolicy, PolicySource, PolicySourceKind from fixlib.graph import Graph from fixlib.json import sort_json from fixlib.json_bender import Bend, Bender, S, ForallBend, bend @@ -63,7 +63,7 @@ class AwsKmsMultiRegionConfig: @define(eq=False, slots=False) -class AwsKmsKey(AwsResource, BaseAccessKey): +class AwsKmsKey(AwsResource, BaseAccessKey, HasResourcePolicy): kind: ClassVar[str] = "aws_kms_key" _kind_display: ClassVar[str] = "AWS KMS Key" _kind_description: ClassVar[str] = "AWS KMS Key is a managed service that creates and controls cryptographic keys used to protect data in AWS services and applications. It generates, stores, and manages keys for encryption and decryption operations. KMS Keys integrate with other AWS services, providing a centralized system for key management and helping users meet compliance requirements for data security and access control." # fmt: skip @@ -121,6 +121,11 @@ class AwsKmsKey(AwsResource, BaseAccessKey): kms_key_rotation_enabled: Optional[bool] = field(default=None) kms_key_policy: Optional[Json] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.kms_key_policy: + return [] + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.kms_key_policy or {})] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/fix_plugin_aws/resource/lambda_.py b/plugins/aws/fix_plugin_aws/resource/lambda_.py index aaacd1482b..9d9551d566 100644 --- a/plugins/aws/fix_plugin_aws/resource/lambda_.py +++ b/plugins/aws/fix_plugin_aws/resource/lambda_.py @@ -2,72 +2,32 @@ import json as json_p import logging import re -from typing import ClassVar, Dict, Optional, List, Type, Any +from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any from attrs import define, field from fix_plugin_aws.aws_client import AwsClient -from fix_plugin_aws.resource.apigateway import AwsApiGatewayRestApi, AwsApiGatewayResource from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, parse_json from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory from fix_plugin_aws.resource.ec2 import AwsEc2Subnet, AwsEc2SecurityGroup, AwsEc2Vpc from fix_plugin_aws.resource.kms import AwsKmsKey from fixlib.baseresources import ( BaseServerlessFunction, + HasResourcePolicy, MetricName, ModelReference, + PolicySource, + PolicySourceKind, ) from fixlib.graph import Graph from fixlib.json_bender import Bender, S, Bend, ForallBend, F, bend from fixlib.types import Json +from fixlib.json import sort_json log = logging.getLogger("fix.plugins.aws") service_name = "lambda" -@define(eq=False, slots=False) -class AwsLambdaPolicyStatement: - kind: ClassVar[str] = "aws_lambda_policy_statement" - kind_display: ClassVar[str] = "AWS Lambda Policy Statement" - kind_description: ClassVar[str] = ( - "Lambda Policy Statements are used to define permissions for AWS Lambda" - " functions, specifying what actions can be performed and by whom." - ) - mapping: ClassVar[Dict[str, Bender]] = { - "sid": S("Sid"), - "effect": S("Effect"), - "principal": S("Principal"), - "action": S("Action"), - "resource": S("Resource"), - "condition": S("Condition"), - } - sid: Optional[str] = field(default=None) - effect: Optional[str] = field(default=None) - principal: Optional[Any] = field(default=None) - action: Optional[Any] = field(default=None) - resource: Optional[Any] = field(default=None) - condition: Optional[Any] = field(default=None) - - -@define(eq=False, slots=False) -class AwsLambdaPolicy: - kind: ClassVar[str] = "aws_lambda_policy" - kind_display: ClassVar[str] = "AWS Lambda Policy" - kind_description: ClassVar[str] = ( - "AWS Lambda Policies are permissions policies that determine what actions a" - " Lambda function can take and what resources it can access within the AWS" - " environment." - ) - mapping: ClassVar[Dict[str, Bender]] = { - "id": S("Id"), - "version": S("Version"), - "statement": S("Statement") >> ForallBend(AwsLambdaPolicyStatement.mapping), - } - id: Optional[str] = field(default=None) - version: Optional[str] = field(default=None) - statement: Optional[List[AwsLambdaPolicyStatement]] = field(default=None) - - @define(eq=False, slots=False) class AwsLambdaEnvironmentError: kind: ClassVar[str] = "aws_lambda_environment_error" @@ -230,7 +190,7 @@ class AwsLambdaFunctionUrlConfig: @define(eq=False, slots=False) -class AwsLambdaFunction(AwsResource, BaseServerlessFunction): +class AwsLambdaFunction(AwsResource, BaseServerlessFunction, HasResourcePolicy): kind: ClassVar[str] = "aws_lambda_function" _kind_display: ClassVar[str] = "AWS Lambda Function" _aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/lambda/home?region={region}#/functions/{FunctionName}", "arn_tpl": "arn:{partition}:lambda:{region}:{account}:function/{name}"} # fmt: skip @@ -320,9 +280,15 @@ class AwsLambdaFunction(AwsResource, BaseServerlessFunction): function_signing_job_arn: Optional[str] = field(default=None) function_architectures: List[str] = field(factory=list) function_ephemeral_storage: Optional[int] = field(default=None) - function_policy: Optional[AwsLambdaPolicy] = field(default=None) + function_policy: Optional[Json] = field(default=None) function_url_config: Optional[AwsLambdaFunctionUrlConfig] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.function_policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.function_policy)] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ @@ -342,7 +308,7 @@ def add_tags(function: AwsLambdaFunction) -> None: function.tags = tags def get_policy(function: AwsLambdaFunction) -> None: - if policy := builder.client.get( + if raw_policy := builder.client.get( service_name, "get-policy", expected_errors=["ResourceNotFoundException"], # policy is optional @@ -350,33 +316,8 @@ def get_policy(function: AwsLambdaFunction) -> None: result_name="Policy", ): # policy is defined as string, but it is actually a json object - mapped = bend(AwsLambdaPolicy.mapping, json_p.loads(policy)) # type: ignore - if policy_instance := parse_json(mapped, AwsLambdaPolicy, builder): - function.function_policy = policy_instance - for statement in policy_instance.statement or []: - if ( - statement.principal - and statement.condition - and isinstance(statement.principal, dict) - and statement.principal.get("Service") == "apigateway.amazonaws.com" - and (arn_like := statement.condition.get("ArnLike")) is not None - and (source := arn_like.get("AWS:SourceArn")) is not None - ): - source_arn = source.rsplit(":")[-1] - rest_api_id = source_arn.split("/")[0] - builder.dependant_node( - function, - reverse=True, - clazz=AwsApiGatewayRestApi, - id=rest_api_id, - ) - builder.dependant_node( - function, - reverse=True, - clazz=AwsApiGatewayResource, - api_link=rest_api_id, - resource_path="/" + source_arn.split("/")[-1], - ) + json_policy = json_p.loads(raw_policy) # type: ignore + function.function_policy = sort_json(json_policy, sort_list=True) def get_url_config(function: AwsLambdaFunction) -> None: if config := builder.client.get( diff --git a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py index 7303962d3d..43613c26f5 100644 --- a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py +++ b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py @@ -1,14 +1,16 @@ from datetime import datetime -from typing import ClassVar, Dict, Optional, List, Type, Any +from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any from attrs import define, field from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import ModelReference +from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind from fixlib.json_bender import Bender, S, Bend from fixlib.types import Json +from json import loads as json_loads +from fixlib.json import sort_json service_name = "secretsmanager" @@ -27,7 +29,7 @@ class AwsSecretsManagerRotationRulesType: @define(eq=False, slots=False) -class AwsSecretsManagerSecret(AwsResource): +class AwsSecretsManagerSecret(HasResourcePolicy, AwsResource): kind: ClassVar[str] = "aws_secretsmanager_secret" api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "list-secrets", "SecretList") _kind_display: ClassVar[str] = "AWS Secrets Manager Secret" @@ -72,6 +74,38 @@ class AwsSecretsManagerSecret(AwsResource): owning_service: Optional[str] = field(default=None, metadata={"description": "Returns the name of the service that created the secret."}) # fmt: skip created_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time when a secret was created."}) # fmt: skip primary_region: Optional[str] = field(default=None, metadata={"description": "The Region where Secrets Manager originated the secret."}) # fmt: skip + policy: Optional[Json] = field(default=None) + + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.policy)] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return [ + cls.api_spec, + AwsApiSpec(service_name, "get-resource-policy"), + ] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + + def get_policy(secret: AwsSecretsManagerSecret) -> None: + if raw_policy := builder.client.get( + service_name, + "get-resource-policy", + expected_errors=["ResourceNotFoundException"], # policy is optional + SecretId=secret.id, + result_name="ResourcePolicy", + ): + secret.policy = sort_json(json_loads(raw_policy), sort_list=True) # type: ignore + + for js in json: + if instance := cls.from_api(js, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, get_policy, instance) def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: if kms_key_id := source.get("KmsKeyId"): diff --git a/plugins/aws/fix_plugin_aws/resource/sns.py b/plugins/aws/fix_plugin_aws/resource/sns.py index 141b1547c6..b7ad627cd2 100644 --- a/plugins/aws/fix_plugin_aws/resource/sns.py +++ b/plugins/aws/fix_plugin_aws/resource/sns.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import ClassVar, Dict, List, Optional, Type, Any +from typing import ClassVar, Dict, List, Optional, Tuple, Type, Any from attrs import define, field from fix_plugin_aws.aws_client import AwsClient @@ -8,7 +8,7 @@ from fix_plugin_aws.resource.iam import AwsIamRole from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import EdgeType, MetricName, ModelReference +from fixlib.baseresources import EdgeType, HasResourcePolicy, MetricName, ModelReference, PolicySource, PolicySourceKind from fixlib.graph import Graph from fixlib.json_bender import F, Bender, S, bend, ParseJson, Sorted from fixlib.types import Json @@ -17,7 +17,7 @@ @define(eq=False, slots=False) -class AwsSnsTopic(AwsResource): +class AwsSnsTopic(AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_sns_topic" _kind_display: ClassVar[str] = "AWS SNS Topic" _kind_description: ClassVar[str] = "AWS SNS Topic is a messaging service that facilitates communication between distributed systems, applications, and microservices. It implements a publish-subscribe model, where publishers send messages to topics and subscribers receive those messages. SNS supports multiple protocols for message delivery, including HTTP, email, SMS, and mobile push notifications, making it useful for various notification scenarios." # fmt: skip @@ -58,6 +58,12 @@ class AwsSnsTopic(AwsResource): topic_fifo_topic: Optional[bool] = field(default=None) topic_content_based_deduplication: Optional[bool] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.topic_policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.topic_policy)] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/fix_plugin_aws/resource/sqs.py b/plugins/aws/fix_plugin_aws/resource/sqs.py index 52e1db3472..1632ca5bf3 100644 --- a/plugins/aws/fix_plugin_aws/resource/sqs.py +++ b/plugins/aws/fix_plugin_aws/resource/sqs.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import ClassVar, Dict, List, Optional, Type, Any +from typing import ClassVar, Dict, List, Optional, Tuple, Type, Any from attrs import define, field @@ -8,7 +8,14 @@ from fix_plugin_aws.resource.base import AwsApiSpec, AwsResource, GraphBuilder from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory from fix_plugin_aws.resource.kms import AwsKmsKey -from fixlib.baseresources import BaseQueue, MetricName, ModelReference +from fixlib.baseresources import ( + BaseQueue, + HasResourcePolicy, + MetricName, + ModelReference, + PolicySource, + PolicySourceKind, +) from fixlib.graph import Graph from fixlib.json_bender import F, Bender, S, AsInt, AsBool, Bend, ParseJson, Sorted from fixlib.types import Json @@ -35,7 +42,7 @@ class AwsSqsRedrivePolicy: @define(eq=False, slots=False) -class AwsSqsQueue(AwsResource, BaseQueue): +class AwsSqsQueue(AwsResource, BaseQueue, HasResourcePolicy): kind: ClassVar[str] = "aws_sqs_queue" _kind_display: ClassVar[str] = "AWS SQS Queue" _kind_description: ClassVar[str] = "AWS SQS Queue is a managed message queuing service that facilitates communication between distributed system components. It stores messages from producers and delivers them to consumers, ensuring reliable data transfer. SQS supports multiple messaging patterns, including point-to-point and publish-subscribe, and handles message retention, delivery, and deletion. It integrates with other AWS services for building decoupled applications." # fmt: skip @@ -96,6 +103,12 @@ class AwsSqsQueue(AwsResource, BaseQueue): sqs_receive_message_wait_time_seconds: Optional[int] = field(default=None) sqs_managed_sse_enabled: Optional[bool] = field(default=None) + def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: + if not self.sqs_policy: + return [] + + return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.sqs_policy)] + @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: return [ diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index 9c2b747013..5d19ae5c2c 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -35,7 +35,7 @@ def count_kind(clazz: Type[AwsResource]) -> int: assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges assert count_kind(AwsResource) == 261 - assert len(account_collector.graph.edges) == 579 + assert len(account_collector.graph.edges) == 575 assert len(account_collector.graph.deferred_edges) == 2 for node in account_collector.graph.nodes: if isinstance(node, AwsRegion): diff --git a/plugins/aws/test/resources/files/secretsmanager/get-resource-policy__foo.json b/plugins/aws/test/resources/files/secretsmanager/get-resource-policy__foo.json new file mode 100644 index 0000000000..71b4d4d733 --- /dev/null +++ b/plugins/aws/test/resources/files/secretsmanager/get-resource-policy__foo.json @@ -0,0 +1,5 @@ +{ + "ARN": "foo", + "Name": "bar", + "ResourcePolicy": "{\"Version\":\"2012-10-17\",\"Statement\":[]}" +} \ No newline at end of file diff --git a/plugins/aws/test/resources/lambda_test.py b/plugins/aws/test/resources/lambda_test.py index 780ccbabf5..496d8bd681 100644 --- a/plugins/aws/test/resources/lambda_test.py +++ b/plugins/aws/test/resources/lambda_test.py @@ -1,33 +1,11 @@ -from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction, AwsLambdaPolicy +from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction from fixlib.graph import Graph -from fixlib.json import from_json from test.resources import round_trip_for from typing import Any, cast from types import SimpleNamespace from fix_plugin_aws.aws_client import AwsClient -def test_regression_lamda_get_policy() -> None: - value_to_read = { - "policy": { - "id": "default", - "version": "2012-10-17", - "statement": [ - { - "sid": "StackSet-AWSControlTower-ALCD-LZ-resource-owner-tag", - "effect": "Allow", - "principal": {"Service": "events.amazonaws.com"}, - "action": "lambda:InvokeFunction", - "resource": "arn:aws:lambda:eu-central-1:test:function:aws-controltower-owner-tagging-func", - "condition": None, - } - ], - }, - "policy_revision_id": "b3f179eb-569b-4ea2-8ec4-4324609b0694", - } - from_json(value_to_read, AwsLambdaPolicy) - - def test_lambda() -> None: first, graph = round_trip_for(AwsLambdaFunction) assert len(graph.resources_of(AwsLambdaFunction)) == 2 From f68420c360090d8f92251c7d4ba94d4de36263e0 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 12:49:19 +0000 Subject: [PATCH 43/47] pr feedback --- fixlib/fixlib/baseresources.py | 18 ++--- plugins/aws/fix_plugin_aws/access_edges.py | 32 ++++----- plugins/aws/fix_plugin_aws/collector.py | 3 +- plugins/aws/fix_plugin_aws/configuration.py | 5 +- plugins/aws/fix_plugin_aws/resource/backup.py | 2 +- .../aws/fix_plugin_aws/resource/dynamodb.py | 4 +- plugins/aws/fix_plugin_aws/resource/ecr.py | 2 +- plugins/aws/fix_plugin_aws/resource/efs.py | 2 +- plugins/aws/fix_plugin_aws/resource/kms.py | 2 +- .../aws/fix_plugin_aws/resource/lambda_.py | 2 +- plugins/aws/fix_plugin_aws/resource/s3.py | 2 +- .../fix_plugin_aws/resource/secretsmanager.py | 2 +- plugins/aws/fix_plugin_aws/resource/sns.py | 2 +- plugins/aws/fix_plugin_aws/resource/sqs.py | 2 +- plugins/aws/test/acccess_edges_test.py | 72 +++++++++---------- 15 files changed, 77 insertions(+), 75 deletions(-) diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 940f0f2e69..11b1028b68 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1614,9 +1614,9 @@ def delete(self, graph: Any) -> bool: class PolicySourceKind(StrEnum): - Principal = "principal" # e.g. IAM user, attached policy - Group = "group" # policy comes from an IAM group - Resource = "resource" # e.g. s3 bucket policy + principal = "principal" # e.g. IAM user, attached policy + group = "group" # policy comes from an IAM group + resource = "resource" # e.g. s3 bucket policy ResourceConstraint = str @@ -1671,12 +1671,12 @@ def has_no_condititons(self) -> bool: class PermissionLevel(StrEnum): - List = "list" - Read = "read" - Tagging = "tagging" - Write = "write" - PermissionManagement = "permission" - Unknown = "unknown" # in case a resource is not in the levels database + list = "list" + read = "read" + tagging = "tagging" + write = "write" + permission_management = "permission" + unknown = "unknown" # in case a resource is not in the levels database @frozen diff --git a/plugins/aws/fix_plugin_aws/access_edges.py b/plugins/aws/fix_plugin_aws/access_edges.py index 362ec34e0e..0a11a8d6d8 100644 --- a/plugins/aws/fix_plugin_aws/access_edges.py +++ b/plugins/aws/fix_plugin_aws/access_edges.py @@ -171,7 +171,7 @@ def check_statement_match( # shortcuts for known AWS managed policies if source_arn == "arn:aws:iam::aws:policy/ReadOnlyAccess": action_level = get_action_level(action) - if action_level in [PermissionLevel.Read or PermissionLevel.List]: + if action_level in [PermissionLevel.read or PermissionLevel.list]: action_match = True else: action_match = False @@ -476,24 +476,24 @@ def get_action_level(action: str) -> PermissionLevel: level = "" action_data = get_action_data(service, action_name) if not action_data: - return PermissionLevel.Unknown + return PermissionLevel.unknown if len(action_data[service]) > 0: for info in action_data[service]: if action == info["action"]: level = info["access_level"] break if level == "List": - return PermissionLevel.List + return PermissionLevel.list elif level == "Read": - return PermissionLevel.Read + return PermissionLevel.read elif level == "Tagging": - return PermissionLevel.Tagging + return PermissionLevel.tagging elif level == "Write": - return PermissionLevel.Write + return PermissionLevel.write elif level == "Permissions management": - return PermissionLevel.PermissionManagement + return PermissionLevel.permission_management else: - return PermissionLevel.Unknown + return PermissionLevel.unknown # logic according to https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html @@ -692,7 +692,7 @@ def _init_principals(self) -> None: def _get_user_based_policies(self, principal: AwsIamUser) -> List[Tuple[PolicySource, PolicyDocument]]: inline_policies = [ ( - PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), + PolicySource(kind=PolicySourceKind.principal, uri=principal.arn or ""), PolicyDocument(policy.policy_document), ) for policy in principal.user_policies @@ -705,7 +705,7 @@ def _get_user_based_policies(self, principal: AwsIamUser) -> List[Tuple[PolicySo if doc := to_node.policy_document_json(): attached_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), + PolicySource(kind=PolicySourceKind.principal, uri=to_node.arn or ""), PolicyDocument(doc), ) ) @@ -717,7 +717,7 @@ def _get_user_based_policies(self, principal: AwsIamUser) -> List[Tuple[PolicySo if policy.policy_document: group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, uri=group.arn or ""), + PolicySource(kind=PolicySourceKind.group, uri=group.arn or ""), PolicyDocument(policy.policy_document), ) ) @@ -727,7 +727,7 @@ def _get_user_based_policies(self, principal: AwsIamUser) -> List[Tuple[PolicySo if doc := group_successor.policy_document_json(): group_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, uri=group_successor.arn or ""), + PolicySource(kind=PolicySourceKind.group, uri=group_successor.arn or ""), PolicyDocument(doc), ) ) @@ -738,7 +738,7 @@ def _get_group_based_policies(self, principal: AwsIamGroup) -> List[Tuple[Policy # not really a principal, but could be useful to have access edges for groups inline_policies = [ ( - PolicySource(kind=PolicySourceKind.Group, uri=principal.arn or ""), + PolicySource(kind=PolicySourceKind.group, uri=principal.arn or ""), PolicyDocument(policy.policy_document), ) for policy in principal.group_policies @@ -751,7 +751,7 @@ def _get_group_based_policies(self, principal: AwsIamGroup) -> List[Tuple[Policy if doc := to_node.policy_document_json(): attached_policies.append( ( - PolicySource(kind=PolicySourceKind.Group, uri=to_node.arn or ""), + PolicySource(kind=PolicySourceKind.group, uri=to_node.arn or ""), PolicyDocument(doc), ) ) @@ -763,7 +763,7 @@ def _get_role_based_policies(self, principal: AwsIamRole) -> List[Tuple[PolicySo for doc in [p.policy_document for p in principal.role_policies if p.policy_document]: inline_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn or ""), + PolicySource(kind=PolicySourceKind.principal, uri=principal.arn or ""), PolicyDocument(doc), ) ) @@ -774,7 +774,7 @@ def _get_role_based_policies(self, principal: AwsIamRole) -> List[Tuple[PolicySo if policy_doc := to_node.policy_document_json(): attached_policies.append( ( - PolicySource(kind=PolicySourceKind.Principal, uri=to_node.arn or ""), + PolicySource(kind=PolicySourceKind.principal, uri=to_node.arn or ""), PolicyDocument(policy_doc), ) ) diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index 8a9e461b3c..bf731d5605 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -255,7 +255,8 @@ def get_last_run() -> Optional[datetime]: log.warning(f"Unexpected node type {node} in graph") raise Exception("Only AWS resources expected") - if global_builder.config.collect_access_edges: + access_edge_collection_enabled = True + if access_edge_collection_enabled and global_builder.config.collect_access_edges: # add access edges log.info(f"[Aws:{self.account.id}] Create access edges.") access_edge_creator = AccessEdgeCreator(global_builder) diff --git a/plugins/aws/fix_plugin_aws/configuration.py b/plugins/aws/fix_plugin_aws/configuration.py index 95ac583dfb..c88645745b 100644 --- a/plugins/aws/fix_plugin_aws/configuration.py +++ b/plugins/aws/fix_plugin_aws/configuration.py @@ -1,4 +1,5 @@ import logging +from re import T import threading import time import uuid @@ -269,8 +270,8 @@ class AwsConfig: metadata={"description": "Collect resource usage metrics via CloudWatch, enabled by default"}, ) collect_access_edges: Optional[bool] = field( - default=False, - metadata={"description": "Collect IAM access edges, disabled by default"}, + default=True, + metadata={"description": "Collect IAM access edges, enabled by default"}, ) @staticmethod diff --git a/plugins/aws/fix_plugin_aws/resource/backup.py b/plugins/aws/fix_plugin_aws/resource/backup.py index dbbbcd3e32..0e52e23fb5 100644 --- a/plugins/aws/fix_plugin_aws/resource/backup.py +++ b/plugins/aws/fix_plugin_aws/resource/backup.py @@ -345,7 +345,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.vault_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, uri=self.arn or ""), self.vault_policy or {})] + return [(PolicySource(PolicySourceKind.resource, uri=self.arn or ""), self.vault_policy or {})] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index ac678969ff..ad808293be 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -422,7 +422,7 @@ class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: if not self.dynamodb_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.dynamodb_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy or {})] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: @@ -531,7 +531,7 @@ class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): dynamodb_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.dynamodb_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy or {})] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/ecr.py b/plugins/aws/fix_plugin_aws/resource/ecr.py index 7cea09aee6..584d41601b 100644 --- a/plugins/aws/fix_plugin_aws/resource/ecr.py +++ b/plugins/aws/fix_plugin_aws/resource/ecr.py @@ -61,7 +61,7 @@ class AwsEcrRepository(AwsResource, HasResourcePolicy): def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: if not self.repository_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.repository_arn or ""), self.repository_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.repository_arn or ""), self.repository_policy or {})] @classmethod def collect_resources(cls, builder: GraphBuilder) -> None: diff --git a/plugins/aws/fix_plugin_aws/resource/efs.py b/plugins/aws/fix_plugin_aws/resource/efs.py index 1dced62eda..927b5c7c5a 100644 --- a/plugins/aws/fix_plugin_aws/resource/efs.py +++ b/plugins/aws/fix_plugin_aws/resource/efs.py @@ -120,7 +120,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.file_system_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.file_system_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.file_system_policy or {})] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/kms.py b/plugins/aws/fix_plugin_aws/resource/kms.py index ae019b4dd1..fea96b5b14 100644 --- a/plugins/aws/fix_plugin_aws/resource/kms.py +++ b/plugins/aws/fix_plugin_aws/resource/kms.py @@ -124,7 +124,7 @@ class AwsKmsKey(AwsResource, BaseAccessKey, HasResourcePolicy): def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: if not self.kms_key_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.kms_key_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.kms_key_policy or {})] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/lambda_.py b/plugins/aws/fix_plugin_aws/resource/lambda_.py index 9d9551d566..3ff2445b09 100644 --- a/plugins/aws/fix_plugin_aws/resource/lambda_.py +++ b/plugins/aws/fix_plugin_aws/resource/lambda_.py @@ -287,7 +287,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.function_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.function_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.function_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index cef9165212..cde55e00ee 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -195,7 +195,7 @@ class AwsS3Bucket(AwsResource, BaseBucket, HasResourcePolicy): def resource_policy(self, builder: GraphBuilder) -> List[Tuple[PolicySource, Dict[str, Any]]]: assert self.arn - return [(PolicySource(PolicySourceKind.Resource, self.arn), self.bucket_policy)] if self.bucket_policy else [] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.bucket_policy)] if self.bucket_policy else [] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py index 43613c26f5..632d8dc220 100644 --- a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py +++ b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py @@ -80,7 +80,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/sns.py b/plugins/aws/fix_plugin_aws/resource/sns.py index b7ad627cd2..6b6448e590 100644 --- a/plugins/aws/fix_plugin_aws/resource/sns.py +++ b/plugins/aws/fix_plugin_aws/resource/sns.py @@ -62,7 +62,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.topic_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.topic_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.topic_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/sqs.py b/plugins/aws/fix_plugin_aws/resource/sqs.py index 1632ca5bf3..dbadc4e5d1 100644 --- a/plugins/aws/fix_plugin_aws/resource/sqs.py +++ b/plugins/aws/fix_plugin_aws/resource/sqs.py @@ -107,7 +107,7 @@ def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, An if not self.sqs_policy: return [] - return [(PolicySource(PolicySourceKind.Resource, self.arn or ""), self.sqs_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.sqs_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/test/acccess_edges_test.py b/plugins/aws/test/acccess_edges_test.py index ac31e3a35e..f3a2c5c375 100644 --- a/plugins/aws/test/acccess_edges_test.py +++ b/plugins/aws/test/acccess_edges_test.py @@ -178,7 +178,7 @@ def test_explicit_deny_in_identity_policy() -> None: "Statement": [{"Effect": "Deny", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::example-bucket/*"}], } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=principal.arn), policy_document)] permission_boundaries: List[PolicyDocument] = [] service_control_policy_levels: List[List[PolicyDocument]] = [] @@ -213,7 +213,7 @@ def test_explicit_deny_with_condition_in_identity_policy() -> None: ], } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=principal.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=principal.arn), policy_document)] request_context = IamRequestContext( principal=principal, @@ -316,7 +316,7 @@ def test_explicit_deny_in_resource_policy() -> None: } policy_document = PolicyDocument(policy_json) resource_based_policies = [ - (PolicySource(kind=PolicySourceKind.Resource, uri="arn:aws:s3:::example-bucket"), policy_document) + (PolicySource(kind=PolicySourceKind.resource, uri="arn:aws:s3:::example-bucket"), policy_document) ] resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") @@ -351,7 +351,7 @@ def test_explicit_deny_with_condition_in_resource_policy() -> None: } policy_document = PolicyDocument(policy_json) resource_based_policies = [ - (PolicySource(kind=PolicySourceKind.Resource, uri="arn:aws:s3:::example-bucket"), policy_document) + (PolicySource(kind=PolicySourceKind.resource, uri="arn:aws:s3:::example-bucket"), policy_document) ] resource = AwsResource(id="some-resource", arn="arn:aws:s3:::example-bucket/object.txt") @@ -381,7 +381,7 @@ def test_compute_permissions_user_inline_policy_allow() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -390,10 +390,10 @@ def test_compute_permissions_user_inline_policy_allow() -> None: permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == PermissionLevel.List + assert permissions[0].level == PermissionLevel.list assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] - assert s.source.kind == PolicySourceKind.Principal + assert s.source.kind == PolicySourceKind.principal assert s.source.uri == user.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) @@ -420,7 +420,7 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -429,10 +429,10 @@ def test_compute_permissions_user_inline_policy_allow_with_conditions() -> None: permissions = compute_permissions(resource=bucket, iam_context=request_context, resource_based_policies=[]) assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == PermissionLevel.List + assert permissions[0].level == PermissionLevel.list assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] - assert s.source.kind == PolicySourceKind.Principal + assert s.source.kind == PolicySourceKind.principal assert s.source.uri == user.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) assert s.conditions @@ -458,7 +458,7 @@ def test_compute_permissions_user_inline_policy_deny() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -491,7 +491,7 @@ def test_compute_permissions_user_inline_policy_deny_with_condition() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), policy_document)] request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -536,8 +536,8 @@ def test_deny_overrides_allow() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -582,8 +582,8 @@ def test_deny_different_action_does_not_override_allow() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -631,8 +631,8 @@ def test_deny_overrides_allow_with_condition() -> None: allow_policy_document = PolicyDocument(allow_policy_json) identity_policies = [ - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), deny_policy_document), - (PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), allow_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), deny_policy_document), + (PolicySource(kind=PolicySourceKind.principal, uri=user.arn), allow_policy_document), ] request_context = IamRequestContext( @@ -644,10 +644,10 @@ def test_deny_overrides_allow_with_condition() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == PermissionLevel.List + assert p.level == PermissionLevel.list assert len(p.scopes) == 1 s = p.scopes[0] - assert s.source.kind == PolicySourceKind.Principal + assert s.source.kind == PolicySourceKind.principal assert s.source.uri == user.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) assert s.conditions @@ -678,7 +678,7 @@ def test_compute_permissions_resource_based_policy_allow() -> None: principal=user, identity_policies=[], permission_boundaries=[], service_control_policy_levels=[] ) - resource_based_policies = [(PolicySource(kind=PolicySourceKind.Resource, uri=bucket.arn), policy_document)] + resource_based_policies = [(PolicySource(kind=PolicySourceKind.resource, uri=bucket.arn), policy_document)] permissions = compute_permissions( resource=bucket, iam_context=request_context, resource_based_policies=resource_based_policies @@ -687,10 +687,10 @@ def test_compute_permissions_resource_based_policy_allow() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == PermissionLevel.List + assert p.level == PermissionLevel.list assert len(p.scopes) == 1 s = p.scopes[0] - assert s.source.kind == PolicySourceKind.Resource + assert s.source.kind == PolicySourceKind.resource assert s.source.uri == bucket.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) @@ -728,7 +728,7 @@ def test_compute_permissions_permission_boundary_restrict() -> None: } permission_boundary_document = PolicyDocument(permission_boundary_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), identity_policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), identity_policy_document)] permission_boundaries = [permission_boundary_document] @@ -744,10 +744,10 @@ def test_compute_permissions_permission_boundary_restrict() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == PermissionLevel.List + assert p.level == PermissionLevel.list assert len(p.scopes) == 1 s = p.scopes[0] - assert s.source.kind == PolicySourceKind.Principal + assert s.source.kind == PolicySourceKind.principal assert s.source.uri == user.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) @@ -779,7 +779,7 @@ def test_compute_permissions_scp_deny() -> None: } scp_policy_document = PolicyDocument(scp_policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=user.arn), identity_policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=user.arn), identity_policy_document)] service_control_policy_levels = [[scp_policy_document]] @@ -812,7 +812,7 @@ def test_compute_permissions_user_with_group_policies() -> None: identity_policies = [] - identity_policies.append((PolicySource(kind=PolicySourceKind.Group, uri=group.arn), group_policy_document)) + identity_policies.append((PolicySource(kind=PolicySourceKind.group, uri=group.arn), group_policy_document)) request_context = IamRequestContext( principal=user, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -823,10 +823,10 @@ def test_compute_permissions_user_with_group_policies() -> None: assert len(permissions) == 1 p = permissions[0] assert p.action == "s3:ListBucket" - assert p.level == PermissionLevel.List + assert p.level == PermissionLevel.list assert len(p.scopes) == 1 s = p.scopes[0] - assert s.source.kind == PolicySourceKind.Group + assert s.source.kind == PolicySourceKind.group assert s.source.uri == group.arn assert s.constraints == (bucket.arn,) @@ -864,7 +864,7 @@ def test_compute_permissions_group_inline_policy_allow() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Group, uri=group.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.group, uri=group.arn), policy_document)] request_context = IamRequestContext( principal=group, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -874,10 +874,10 @@ def test_compute_permissions_group_inline_policy_allow() -> None: assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == PermissionLevel.List + assert permissions[0].level == PermissionLevel.list assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] - assert s.source.kind == PolicySourceKind.Group + assert s.source.kind == PolicySourceKind.group assert s.source.uri == group.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) @@ -901,7 +901,7 @@ def test_compute_permissions_role_inline_policy_allow() -> None: } policy_document = PolicyDocument(policy_json) - identity_policies = [(PolicySource(kind=PolicySourceKind.Principal, uri=role.arn), policy_document)] + identity_policies = [(PolicySource(kind=PolicySourceKind.principal, uri=role.arn), policy_document)] request_context = IamRequestContext( principal=role, identity_policies=identity_policies, permission_boundaries=[], service_control_policy_levels=[] @@ -911,9 +911,9 @@ def test_compute_permissions_role_inline_policy_allow() -> None: assert len(permissions) == 1 assert permissions[0].action == "s3:ListBucket" - assert permissions[0].level == PermissionLevel.List + assert permissions[0].level == PermissionLevel.list assert len(permissions[0].scopes) == 1 s = permissions[0].scopes[0] - assert s.source.kind == PolicySourceKind.Principal + assert s.source.kind == PolicySourceKind.principal assert s.source.uri == role.arn assert s.constraints == ("arn:aws:s3:::my-test-bucket",) From 0777803d8478e2fa8ad0fcab9957f9db8d00b3a7 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 12:56:32 +0000 Subject: [PATCH 44/47] set access edges flag to disabled --- plugins/aws/fix_plugin_aws/collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index bf731d5605..5a3debc981 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -255,7 +255,7 @@ def get_last_run() -> Optional[datetime]: log.warning(f"Unexpected node type {node} in graph") raise Exception("Only AWS resources expected") - access_edge_collection_enabled = True + access_edge_collection_enabled = False if access_edge_collection_enabled and global_builder.config.collect_access_edges: # add access edges log.info(f"[Aws:{self.account.id}] Create access edges.") From 06b03e79296d3eaabe590d5c57f51168a3c98c48 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 13:14:30 +0000 Subject: [PATCH 45/47] lintex fix --- plugins/aws/fix_plugin_aws/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/aws/fix_plugin_aws/configuration.py b/plugins/aws/fix_plugin_aws/configuration.py index c88645745b..6d8f7a544f 100644 --- a/plugins/aws/fix_plugin_aws/configuration.py +++ b/plugins/aws/fix_plugin_aws/configuration.py @@ -1,5 +1,4 @@ import logging -from re import T import threading import time import uuid From c32fdd1531493656ed95517322579b54bf057d0c Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 13:32:16 +0000 Subject: [PATCH 46/47] a bit of more fixes --- plugins/aws/fix_plugin_aws/resource/dynamodb.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index ad808293be..9b3f00f209 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -422,7 +422,7 @@ class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: if not self.dynamodb_policy: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: @@ -531,7 +531,10 @@ class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): dynamodb_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy or {})] + if not self.dynamodb_policy: + return [] + + return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: From dd181a0a295622d982a2555c1c7751b1c6223f31 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Tue, 8 Oct 2024 13:58:09 +0000 Subject: [PATCH 47/47] remove default arn --- plugins/aws/fix_plugin_aws/resource/backup.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/dynamodb.py | 8 ++++---- plugins/aws/fix_plugin_aws/resource/efs.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/kms.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/lambda_.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/secretsmanager.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/sns.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/sqs.py | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/plugins/aws/fix_plugin_aws/resource/backup.py b/plugins/aws/fix_plugin_aws/resource/backup.py index 0e52e23fb5..cbb18e9834 100644 --- a/plugins/aws/fix_plugin_aws/resource/backup.py +++ b/plugins/aws/fix_plugin_aws/resource/backup.py @@ -342,10 +342,10 @@ class AwsBackupVault(BackupResourceTaggable, AwsResource, HasResourcePolicy): vault_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.vault_policy: + if not self.vault_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, uri=self.arn or ""), self.vault_policy or {})] + return [(PolicySource(PolicySourceKind.resource, uri=self.arn), self.vault_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index 9b3f00f209..c7bbfbf4fb 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -420,9 +420,9 @@ class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): dynamodb_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.dynamodb_policy: + if not self.dynamodb_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.dynamodb_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: @@ -531,10 +531,10 @@ class AwsDynamoDbGlobalTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): dynamodb_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.dynamodb_policy: + if not self.dynamodb_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.dynamodb_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.dynamodb_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/efs.py b/plugins/aws/fix_plugin_aws/resource/efs.py index 927b5c7c5a..50c7fc10b5 100644 --- a/plugins/aws/fix_plugin_aws/resource/efs.py +++ b/plugins/aws/fix_plugin_aws/resource/efs.py @@ -117,10 +117,10 @@ class AwsEfsFileSystem(EfsTaggable, AwsResource, BaseNetworkShare, HasResourcePo file_system_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.file_system_policy: + if not self.file_system_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.file_system_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.file_system_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/kms.py b/plugins/aws/fix_plugin_aws/resource/kms.py index fea96b5b14..34db963bd9 100644 --- a/plugins/aws/fix_plugin_aws/resource/kms.py +++ b/plugins/aws/fix_plugin_aws/resource/kms.py @@ -122,9 +122,9 @@ class AwsKmsKey(AwsResource, BaseAccessKey, HasResourcePolicy): kms_key_policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.kms_key_policy: + if not self.kms_key_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.kms_key_policy or {})] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.kms_key_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/lambda_.py b/plugins/aws/fix_plugin_aws/resource/lambda_.py index 3ff2445b09..caa2da3f6b 100644 --- a/plugins/aws/fix_plugin_aws/resource/lambda_.py +++ b/plugins/aws/fix_plugin_aws/resource/lambda_.py @@ -284,10 +284,10 @@ class AwsLambdaFunction(AwsResource, BaseServerlessFunction, HasResourcePolicy): function_url_config: Optional[AwsLambdaFunctionUrlConfig] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.function_policy: + if not self.function_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.function_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.function_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py index 632d8dc220..e79f52a172 100644 --- a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py +++ b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py @@ -77,10 +77,10 @@ class AwsSecretsManagerSecret(HasResourcePolicy, AwsResource): policy: Optional[Json] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.policy: + if not self.policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/sns.py b/plugins/aws/fix_plugin_aws/resource/sns.py index 6b6448e590..6a22292149 100644 --- a/plugins/aws/fix_plugin_aws/resource/sns.py +++ b/plugins/aws/fix_plugin_aws/resource/sns.py @@ -59,10 +59,10 @@ class AwsSnsTopic(AwsResource, HasResourcePolicy): topic_content_based_deduplication: Optional[bool] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.topic_policy: + if not self.topic_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.topic_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.topic_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/fix_plugin_aws/resource/sqs.py b/plugins/aws/fix_plugin_aws/resource/sqs.py index 9495fd5487..ebaa8923be 100644 --- a/plugins/aws/fix_plugin_aws/resource/sqs.py +++ b/plugins/aws/fix_plugin_aws/resource/sqs.py @@ -104,10 +104,10 @@ class AwsSqsQueue(AwsResource, BaseQueue, HasResourcePolicy): sqs_managed_sse_enabled: Optional[bool] = field(default=None) def resource_policy(self, builder: Any) -> List[Tuple[PolicySource, Dict[str, Any]]]: - if not self.sqs_policy: + if not self.sqs_policy or not self.arn: return [] - return [(PolicySource(PolicySourceKind.resource, self.arn or ""), self.sqs_policy)] + return [(PolicySource(PolicySourceKind.resource, self.arn), self.sqs_policy)] @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: