From 02e8a6a25edd1e962b44bc9d4cf769e828dc1b03 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Mon, 1 Jul 2024 11:01:43 +0100 Subject: [PATCH] Dev -> Main (#41) * Add initial python for updating home areas * adding comments for future work * Update rbac.py * pre=release * prerelease test * PRERELEASE * release work flow test * pre release * Update rbac.py * clean up home area function * add setuptools requirements * Update setup.py * remove quotes unneeded * Retrofit logging and env dict from rbac uplift (#17) * flexibility * logging * add shorthand options * options for log levels * Update logging.py * Update __init__.py * Nit 824 nit 823 - update user roles and user notes (#18) * new functions and structure * find common entries in both * refactor + python rewrite foruser roles * remove action * remove debugging * start oracle db * add update notes * typo + rm commented code * refactor + comments * Update __init__.py * fix logger duplicates * re format + remove print debugging * log levels + debugging * Update logger.py * fixes requirements * reformat connection for oracle * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * bind by name * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * Update user.py * add handling for user notes * Nit 822 (#19) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt --------- Co-authored-by: Seb Norris * Nit 822 (#20) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py --------- Co-authored-by: Seb Norris * Nit 822 (#21) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac --------- Co-authored-by: Seb Norris * Nit 822 (#22) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac * Update rbac.py --------- Co-authored-by: Seb Norris * Nit 822 (#23) * add CRC user script * add click cmd * add deactivate-crc-users to main group * Update user.py * Update requirements.txt * Update rbac.py * no token needed for rbac * Update rbac.py * ldap config dict or local val --------- Co-authored-by: Seb Norris * Formatting & linting pre commits (#24) * add pre commit * Update readme.md * format * Update tag-and-release.yml * Update pyproject.toml * Update .flake8 * Update .flake8 * use black defualt * format to black defaults * update black to latest * remove boilerplate excludes * update logging and requirements * NIT-854 Add exception handling and add logging where appropriate * NIT-854 fix typos * Apply suggestions from code review Co-authored-by: George Taylor * Update rbac.py * migration to python-ldap - correction on tree deletion (#28) * Merge branch 'main' into dev * Update .flake8 * User expiry script added * Update rbac.py (#31) * init py change added * quotations * add error handling + get the ou path from the cli input/defaults * Update format-python.yml * Nit 1204 ldap data refresh remove passwords (#42) * remove passwords command * formatting * Update user.py * Update user.py * Update user.py * Update format-python.yml * Update format-python.yml * Formatted code with black --line-length 120 * Update format-python.yml --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --------- Co-authored-by: adrianweetman Co-authored-by: Seb Norris Co-authored-by: Andrew Moore Co-authored-by: Andrew Moore <20435317+andrewmooreio@users.noreply.github.com> Co-authored-by: Ijaz Sultan Co-authored-by: IjazMoJ <134407207+IjazMoJ@users.noreply.github.com> Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/format-python.yml | 6 +- cli/__init__.py | 38 +++++- cli/database/__init__.py | 4 +- cli/env.py | 16 +-- cli/git/__init__.py | 16 +-- cli/ldap_cmds/rbac.py | 2 +- cli/ldap_cmds/user.py | 195 +++++++++++++++++++--------- cli/logger.py | 14 +- setup.py | 4 +- 9 files changed, 189 insertions(+), 106 deletions(-) diff --git a/.github/workflows/format-python.yml b/.github/workflows/format-python.yml index 8a96ef6..f5f84e1 100644 --- a/.github/workflows/format-python.yml +++ b/.github/workflows/format-python.yml @@ -1,8 +1,7 @@ name: Format Python on: pull_request: - paths: - - '**.py' + types: [ opened, edited, reopened, synchronize, ready_for_review ] workflow_dispatch: jobs: format: @@ -10,7 +9,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} - name: Format code with black run: | pip install black diff --git a/cli/__init__.py b/cli/__init__.py index f370ca6..1f0ae38 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -186,12 +186,46 @@ def deactivate_crc_users( user_ou, root_dn, ): - cli.ldap.user.deactivate_crc_users( + cli.ldap_cmds.user.deactivate_crc_users( user_ou, root_dn, ) +@click.command() +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def user_expiry(user_ou, root_dn): + cli.ldap_cmds.user.user_expiry(user_ou=user_ou, root_dn=root_dn) + + +@click.command() +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def remove_all_user_passwords(user_ou, root_dn): + cli.ldap_cmds.user.remove_all_user_passwords(user_ou=user_ou, root_dn=root_dn) + + # from cli.ldap import test main_group.add_command(add_roles_to_users) @@ -199,6 +233,8 @@ def deactivate_crc_users( main_group.add_command(update_user_home_areas) main_group.add_command(update_user_roles) main_group.add_command(deactivate_crc_users) +main_group.add_command(user_expiry) +main_group.add_command(remove_all_user_passwords) logger.configure_logging() diff --git a/cli/database/__init__.py b/cli/database/__init__.py index 511366b..0e7dda6 100644 --- a/cli/database/__init__.py +++ b/cli/database/__init__.py @@ -13,7 +13,5 @@ def connection(): log.debug("Created database connection successfully") return conn except Exception as e: - log.exception( - f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}" - ) + log.exception(f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}") raise e diff --git a/cli/env.py b/cli/env.py index 98c815f..0b96acf 100644 --- a/cli/env.py +++ b/cli/env.py @@ -27,9 +27,7 @@ ).replace( "_DICT", "", - ): ast.literal_eval(val) - if "DICT" in key - else val + ): (ast.literal_eval(val) if "DICT" in key else val) for key, val in dotenv_values(".vars").items() if val is not None }, # load development variables @@ -40,9 +38,7 @@ ).replace( "_DICT", "", - ): ast.literal_eval(val) - if "DICT" in key - else val + ): (ast.literal_eval(val) if "DICT" in key else val) for key, val in os.environ.items() if key.startswith("VAR_") and val is not None }, @@ -61,9 +57,7 @@ .replace( "SSM_", "", - ): ast.literal_eval(val) - if "_DICT" in key - else val + ): (ast.literal_eval(val) if "_DICT" in key else val) for key, val in dotenv_values(".secrets").items() if val is not None }, @@ -79,9 +73,7 @@ .replace( "SSM_", "", - ): ast.literal_eval(val) - if "DICT" in key - else val + ): (ast.literal_eval(val) if "DICT" in key else val) for key, val in os.environ.items() if key.startswith("SECRET_") or key.startswith("SSM_") and val is not None }, diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 8a733c9..6b6cf69 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -36,9 +36,7 @@ def get_access_token( headers=headers, ) except Exception as e: - logging.exception( - f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}" - ) + logging.exception(f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}") raise e # extract the token from the response @@ -68,9 +66,7 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception( - f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" - ) + logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") raise e # if there is a token, assume auth is required and use the token and auth_type elif token: @@ -83,9 +79,7 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception( - f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" - ) + logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") raise e # if there is no token, assume auth is not required and clone without else: @@ -97,7 +91,5 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception( - f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" - ) + logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") raise e diff --git a/cli/ldap_cmds/rbac.py b/cli/ldap_cmds/rbac.py index 77a7a57..4d83ec4 100644 --- a/cli/ldap_cmds/rbac.py +++ b/cli/ldap_cmds/rbac.py @@ -439,7 +439,7 @@ def user_ldifs( for file in user_files: records = ldif.LDIFRecordList(open(file, "rb")) records.parse() - + # pprint(records.all_records) # loop through the records for entry in records.all_records: diff --git a/cli/ldap_cmds/user.py b/cli/ldap_cmds/user.py index 93cd96e..3196ec8 100644 --- a/cli/ldap_cmds/user.py +++ b/cli/ldap_cmds/user.py @@ -14,6 +14,7 @@ ) from ldap3 import ( MODIFY_REPLACE, + MODIFY_DELETE, DEREF_NEVER, ) @@ -98,27 +99,17 @@ def parse_user_role_list( return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} -def add_roles_to_user( - username, - roles, - user_ou="ou=Users", - root_dn="dc=moj,dc=com", -): +def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): log.info(f"Adding roles {roles} to user {username}") ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), - env.vars.get("LDAP_USER"), - env.secrets.get("LDAP_BIND_PASSWORD"), + env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") ) for role in roles: try: ldap_connection.add( f"cn={role},cn={username},{user_ou},{root_dn}", attributes={ - "objectClass": [ - "NDRoleAssociation", - "alias", - ], + "objectClass": ["NDRoleAssociation", "alias"], "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", }, ) @@ -136,11 +127,7 @@ def add_roles_to_user( raise Exception(f"Failed to add role {role} to user {username}") -def process_user_roles_list( - user_role_list, - user_ou="ou=Users", - root_dn="dc=moj,dc=com", -): +def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): user_roles = parse_user_role_list(user_role_list) try: for ( @@ -164,15 +151,7 @@ def process_user_roles_list( def update_roles( - roles, - user_ou, - root_dn, - add, - remove, - update_notes, - user_note, - user_filter="(userSector=*)", - role_filter="*", + roles, user_ou, root_dn, add, remove, update_notes, user_note, user_filter="(userSector=*)", role_filter="*" ): if update_notes and (user_note is None or len(user_note) < 1): log.error("User note must be provided when updating notes") @@ -191,12 +170,7 @@ def update_roles( # # Search for users matching the user_filter try: ldap_connection_user_filter.search( - ",".join( - [ - user_ou, - root_dn, - ] - ), + ",".join([user_ou, root_dn]), user_filter, attributes=["cn"], ) @@ -234,12 +208,7 @@ def update_roles( try: ldap_connection_role_filter.search( - ",".join( - [ - user_ou, - root_dn, - ] - ), + ",".join([user_ou, root_dn]), full_role_filter, attributes=["cn"], dereference_aliases=DEREF_NEVER, @@ -263,12 +232,7 @@ def update_roles( # cartesian_product = [(user, role) for user in matched_users for role in roles] - cartesian_product = list( - product( - matched_users, - roles, - ) - ) + cartesian_product = list(product(matched_users, roles)) log.debug("cartesian product: ") log.debug(cartesian_product) @@ -290,11 +254,7 @@ def update_roles( attributes={ "cn": item[1], "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", - "objectClass": [ - "NDRoleAssociation", - "alias", - "top", - ], + "objectClass": ["NDRoleAssociation", "alias", "top"], }, ) except Exception as e: @@ -376,10 +336,7 @@ def update_roles( ######################################### -def deactivate_crc_users( - user_ou, - root_dn, -): +def deactivate_crc_users(user_ou, root_dn): log.info("Deactivating CRC users") ldap_connection = ldap_connect( env.vars.get("LDAP_HOST"), @@ -431,12 +388,7 @@ def deactivate_crc_users( found_users.append(entry.entry_dn for entry in ldap_connection.entries) ldap_connection.search( - ",".join( - [ - user_ou, - root_dn, - ] - ), + ",".join([user_ou, root_dn]), f"(&(!(userHomeArea=*)){user_filter})", attributes=["dn"], ) @@ -477,3 +429,126 @@ def deactivate_crc_users( except: log.exception(f"Failed to update END_DATE for user {user_dn}") connection.close() + + +def user_expiry(user_ou, root_dn): + date_str = f"{datetime.now().strftime('%Y%m%d')}000000Z" + log.info(f"Expiring users with end date {date_str}") + + ldap_connection_lock = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + try: + ldap_connection_lock.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + f"(&(!(pwdAccountLockedTime=*))(|(&(endDate=*)(!(endDate>={date_str})))(&(startDate=*)(!(startDate<={date_str})))))", + attributes=["cn"], + ) + except Exception as e: + log.exception(f"Failed to search for users \n Exception: {e}") + + found_users = [entry.entry_dn for entry in ldap_connection_lock.entries] + log.debug(found_users) + for user in found_users: + try: + ldap_connection_lock.modify( + user, + { + "pwdAccountLockedTime": [ + ( + MODIFY_REPLACE, + ["000001010000Z"], + ) + ] + }, + ) + log.info(f"Locked user {user}") + except Exception as e: + log.exception(f"Failed to unlock user {user} \n Exception: {e}") + + ldap_connection_unlock = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + + try: + ldap_connection_unlock.search( + ",".join([user_ou, root_dn]), + f"(&(pwdAccountLockedTime=000001010000Z)(|(!(endDate=*))(endDate>={date_str}))(|(!(startDate=*))(startDate<={date_str})))", + attributes=["cn"], + ) + except Exception as e: + log.exception(f"Failed to search for users \n Exception: {e}") + + found_users = [entry.entry_dn for entry in ldap_connection_unlock.entries] + log.debug(found_users) + for user in found_users: + try: + ldap_connection_unlock.modify( + user, + { + "pwdAccountLockedTime": [ + ( + MODIFY_DELETE, + ["000001010000Z"], + ) + ] + }, + ) + log.info(f"Unlocked user {user}") + except Exception as e: + log.exception(f"Failed to unlock user {user} \n Exception: {e}") + + +def remove_all_user_passwords(user_ou, root_dn): + log.info("Removing all user passwords") + + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + + user_filter = "(!(cn=AutomatedTestUser))" + + try: + ldap_connection.search( + ",".join([user_ou, root_dn]), + user_filter, + attributes=["cn"], + search_scope="LEVEL", + ) + except Exception as e: + log.exception("Failed to search for users") + raise e + + found_users = [entry.entry_dn for entry in ldap_connection.entries] + log.debug("Users found:") + log.debug(found_users) + + for user in found_users: + try: + ldap_connection.modify( + user, + { + "userPassword": [ + ( + MODIFY_DELETE, + [], + ) + ] + }, + ) + log.info(f"Successfully removed passwd for user {user}, or it didn't have one to begin with") + except Exception as e: + log.exception(f"Failed to remove passwd for user {user}") + raise e + ldap_connection.unbind() diff --git a/cli/logger.py b/cli/logger.py index a76bb6a..16afdf1 100644 --- a/cli/logger.py +++ b/cli/logger.py @@ -15,9 +15,7 @@ def __init__( fmt=format_str, datefmt=datefmt_str, ) - self._secrets_set = set( - cli.env.secrets.values() - ) # Retrieve secrets set here + self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here self.default_msec_format = "%s.%03d" def _filter( @@ -25,10 +23,7 @@ def _filter( s, ): redacted = " ".join( - [ - "*" * len(string) if string in self._secrets_set else string - for string in s.split(" ") - ] + ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] ) return redacted @@ -42,10 +37,7 @@ def format( print("configure_logging") """Configure logging based on environment variables.""" - format = ( - cli.env.vars.get("LOG_FORMAT") - or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" - ) + format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index b28c4dc..10818b2 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,7 @@ standard_pkgs = [r for r in requirements if not r.startswith("git+")] git_pkgs = [r for r in requirements if r.startswith("git+")] -formatted_git_pkgs = [ - f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs -] +formatted_git_pkgs = [f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs] all_reqs = standard_pkgs + formatted_git_pkgs setup(