From 0601d2a2a62cd9ad82ebfdbc7ca14cd25ac32f8e Mon Sep 17 00:00:00 2001 From: Brett Bedevian Date: Wed, 2 Oct 2024 08:52:39 -0400 Subject: [PATCH 1/4] First Pass at a Bulk Reassign --- bulk_reassign_docs.py | 155 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 bulk_reassign_docs.py diff --git a/bulk_reassign_docs.py b/bulk_reassign_docs.py new file mode 100755 index 0000000..3d09c71 --- /dev/null +++ b/bulk_reassign_docs.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +import argparse +import requests +from utils import SigmaClient + + +def get_member_id(client, user_email): + """ Update member + + :access_token: Generated access token + :user_email: Users email + + :returns: ID associated with the user_email + """ + try: + response = client.get( + f"v2/members?search={user_email}" + ) + response.raise_for_status() + # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code + except requests.exceptions.HTTPError as errh: + # The API response's message value is sent in lieu of the full response to display to the user for clarity + raise Exception(errh.response.status_code, f"API message: {errh.response.json()['message']}") + except requests.exceptions.ConnectionError as errc: + raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + except requests.exceptions.Timeout as errt: + raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + except requests.exceptions.RequestException as err: + raise Exception(f"Other Error: {err}, API response: {err.response.text}") + else: + data = response.json() + if len(data['entries']) == 0: + print("No users found with this email:", user_email) + raise SystemExit("Script aborted") + else: + return data['entries'][0]['memberId'] + +def get_member_files(client, user_id): + """ Update member + :user_id: id of current owner + + :returns: an array of file objects + + """ + data = [] + moreResults = True + nextPage = '' + while moreResults: + try: + response = client.get( + f"v2/members/{user_id}/files?typeFilters=workbook&typeFilters=dataset&limit=500{nextPage}" + ) + response.raise_for_status() + + except requests.exceptions.HTTPError as errh: + raise Exception(f"Connection Error: {errh}, API response: {errh.response.text}") + except requests.exceptions.ConnectionError as errc: + raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + except requests.exceptions.Timeout as errt: + raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + except requests.exceptions.RequestException as err: + raise Exception(f"Other Error: {err}, API response: {err.response.text}") + else: + resp = response.json() + data = data + resp['entries'] + if resp['nextPage'] is None: + moreResults = False + else: + pageID = str(resp['nextPage']) + nextPage = f'&page={pageID}' + return data + +def update_file(client, user_id, file_id): + """ Update file + + :access_token: Generated access token + :userId: ID of the new owner + :file_id: File to transfer ownership of + + :returns: Response JSON + + """ + try: + response = client.patch( + f"v2/files/{file_id}", + json={ "ownerId":user_id } + ) + response.raise_for_status() + # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code + except requests.exceptions.HTTPError as errh: + # The API response's message value is sent in lieu of the full response to display to the user for clarity + # Certain error response messages are useful for the user troubleshoot common issues, such as invalid Member Type or New Email already in use + raise Exception(errh.response.status_code, f"API message: {errh.response.json()['message']}") + except requests.exceptions.ConnectionError as errc: + raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + except requests.exceptions.Timeout as errt: + raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + except requests.exceptions.RequestException as err: + raise Exception(f"Other Error: {err}, API response: {err.response.text}") + else: + data = response.json() + print("transfer of document:",file_id, "---", response ) + return data + +def main(): + parser = argparse.ArgumentParser( + description='Transfer all a users documents to another user') + parser.add_argument( + '--env', type=str, required=True, help='env to use: [production | staging].') + parser.add_argument( + '--cloud', type=str, required=True, help='Cloud to use: [aws | gcp | azure]') + parser.add_argument( + '--client_id', type=str, required=True, help='Client ID generated from within Sigma') + parser.add_argument( + '--client_secret', type=str, required=True, help='Client secret API token generated from within Sigma') + parser.add_argument( + '--curr_owner', type=str, required=True, help='Org Member who currently owns the documents') + parser.add_argument( + '--new_owner', type=str, required=True, help='Org Member who you want to transfer the documents to') + + args = parser.parse_args() + client = SigmaClient(args.env, args.cloud, args.client_id, args.client_secret) + + # we need to confirm that both the existing user and new user are valid "check_users" fn + try: + curr_owner_id = get_member_id(client, args.curr_owner) + except Exception as e: + print(f"{e}") + raise SystemExit("Script aborted") + try: + new_owner_id = get_member_id(client, args.new_owner) + except Exception as e: + print(f"{e}") + raise SystemExit("Script aborted") + # get files of curr_user + try: + member_files = get_member_files(client, curr_owner_id) + except Exception as e: + print(f"{e}") + raise SystemExit("Script aborted") + # filter to only docs they own + filtered_arr = [p for p in member_files if p['ownerId'] == curr_owner_id] + # loop through and reassign ownership + for d in filtered_arr: + try: + update_member_response = update_file(client, new_owner_id, d['id']) + except Exception as e: + print(f"{e}") + raise SystemExit("Script aborted") + + + +if __name__ == '__main__': + main() From e7a48178f40245d5f82064fc8e00992085a15162 Mon Sep 17 00:00:00 2001 From: Brett Bedevian Date: Fri, 4 Oct 2024 09:41:12 -0400 Subject: [PATCH 2/4] reformat with formatter and oliver tab suggestion --- bulk_reassign_docs.py | 54 ++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/bulk_reassign_docs.py b/bulk_reassign_docs.py index 3d09c71..9fb15d4 100755 --- a/bulk_reassign_docs.py +++ b/bulk_reassign_docs.py @@ -21,21 +21,26 @@ def get_member_id(client, user_email): # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code except requests.exceptions.HTTPError as errh: # The API response's message value is sent in lieu of the full response to display to the user for clarity - raise Exception(errh.response.status_code, f"API message: {errh.response.json()['message']}") + raise Exception(errh.response.status_code, + f"API message: {errh.response.json()['message']}") except requests.exceptions.ConnectionError as errc: - raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + raise Exception( + f"Connection Error: {errc}, API response: {errc.response.text}") except requests.exceptions.Timeout as errt: - raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + raise Exception( + f"Timeout Error: {errt}, API response: {errt.response.text}") except requests.exceptions.RequestException as err: - raise Exception(f"Other Error: {err}, API response: {err.response.text}") + raise Exception( + f"Other Error: {err}, API response: {err.response.text}") else: data = response.json() if len(data['entries']) == 0: print("No users found with this email:", user_email) - raise SystemExit("Script aborted") + raise SystemExit("Script aborted") else: return data['entries'][0]['memberId'] + def get_member_files(client, user_id): """ Update member :user_id: id of current owner @@ -54,13 +59,17 @@ def get_member_files(client, user_id): response.raise_for_status() except requests.exceptions.HTTPError as errh: - raise Exception(f"Connection Error: {errh}, API response: {errh.response.text}") + raise Exception( + f"Connection Error: {errh}, API response: {errh.response.text}") except requests.exceptions.ConnectionError as errc: - raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + raise Exception( + f"Connection Error: {errc}, API response: {errc.response.text}") except requests.exceptions.Timeout as errt: - raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + raise Exception( + f"Timeout Error: {errt}, API response: {errt.response.text}") except requests.exceptions.RequestException as err: - raise Exception(f"Other Error: {err}, API response: {err.response.text}") + raise Exception( + f"Other Error: {err}, API response: {err.response.text}") else: resp = response.json() data = data + resp['entries'] @@ -71,6 +80,7 @@ def get_member_files(client, user_id): nextPage = f'&page={pageID}' return data + def update_file(client, user_id, file_id): """ Update file @@ -84,25 +94,30 @@ def update_file(client, user_id, file_id): try: response = client.patch( f"v2/files/{file_id}", - json={ "ownerId":user_id } + json={"ownerId": user_id} ) response.raise_for_status() # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code except requests.exceptions.HTTPError as errh: # The API response's message value is sent in lieu of the full response to display to the user for clarity # Certain error response messages are useful for the user troubleshoot common issues, such as invalid Member Type or New Email already in use - raise Exception(errh.response.status_code, f"API message: {errh.response.json()['message']}") + raise Exception(errh.response.status_code, + f"API message: {errh.response.json()['message']}") except requests.exceptions.ConnectionError as errc: - raise Exception(f"Connection Error: {errc}, API response: {errc.response.text}") + raise Exception( + f"Connection Error: {errc}, API response: {errc.response.text}") except requests.exceptions.Timeout as errt: - raise Exception(f"Timeout Error: {errt}, API response: {errt.response.text}") + raise Exception( + f"Timeout Error: {errt}, API response: {errt.response.text}") except requests.exceptions.RequestException as err: - raise Exception(f"Other Error: {err}, API response: {err.response.text}") + raise Exception( + f"Other Error: {err}, API response: {err.response.text}") else: data = response.json() - print("transfer of document:",file_id, "---", response ) + print("transfer of document:", file_id, "---", response) return data + def main(): parser = argparse.ArgumentParser( description='Transfer all a users documents to another user') @@ -120,25 +135,29 @@ def main(): '--new_owner', type=str, required=True, help='Org Member who you want to transfer the documents to') args = parser.parse_args() - client = SigmaClient(args.env, args.cloud, args.client_id, args.client_secret) - + client = SigmaClient(args.env, args.cloud, + args.client_id, args.client_secret) + # we need to confirm that both the existing user and new user are valid "check_users" fn try: curr_owner_id = get_member_id(client, args.curr_owner) except Exception as e: print(f"{e}") raise SystemExit("Script aborted") + try: new_owner_id = get_member_id(client, args.new_owner) except Exception as e: print(f"{e}") raise SystemExit("Script aborted") + # get files of curr_user try: member_files = get_member_files(client, curr_owner_id) except Exception as e: print(f"{e}") raise SystemExit("Script aborted") + # filter to only docs they own filtered_arr = [p for p in member_files if p['ownerId'] == curr_owner_id] # loop through and reassign ownership @@ -150,6 +169,5 @@ def main(): raise SystemExit("Script aborted") - if __name__ == '__main__': main() From e4a3a7da4094fae3e60e4d65df704d2a3ba9ef97 Mon Sep 17 00:00:00 2001 From: Brett Bedevian Date: Mon, 14 Oct 2024 12:30:59 -0400 Subject: [PATCH 3/4] Handle folder arg and small tweaks --- bulk_reassign_docs.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/bulk_reassign_docs.py b/bulk_reassign_docs.py index 9fb15d4..3defb42 100755 --- a/bulk_reassign_docs.py +++ b/bulk_reassign_docs.py @@ -37,6 +37,9 @@ def get_member_id(client, user_email): if len(data['entries']) == 0: print("No users found with this email:", user_email) raise SystemExit("Script aborted") + elif len(data['entries']) > 1: + print("More than one user found with the provided email:", user_email) + raise SystemExit("Script aborted") else: return data['entries'][0]['memberId'] @@ -53,8 +56,9 @@ def get_member_files(client, user_id): nextPage = '' while moreResults: try: + # currently looks for workbooks and datasets but can be changed as needed response = client.get( - f"v2/members/{user_id}/files?typeFilters=workbook&typeFilters=dataset&limit=500{nextPage}" + f"v2/members/{user_id}/files?typeFilters=workbook&typeFilters=dataset&limit=1000{nextPage}" ) response.raise_for_status() @@ -81,7 +85,7 @@ def get_member_files(client, user_id): return data -def update_file(client, user_id, file_id): +def update_file(client, user_id, file_id, folderID): """ Update file :access_token: Generated access token @@ -91,10 +95,13 @@ def update_file(client, user_id, file_id): :returns: Response JSON """ + updateFileBody={"ownerId": user_id} + if folderID: + updateFileBody["parentId"] = folderID try: response = client.patch( f"v2/files/{file_id}", - json={"ownerId": user_id} + json=updateFileBody ) response.raise_for_status() # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code @@ -130,9 +137,11 @@ def main(): parser.add_argument( '--client_secret', type=str, required=True, help='Client secret API token generated from within Sigma') parser.add_argument( - '--curr_owner', type=str, required=True, help='Org Member who currently owns the documents') + '--curr_owner', type=str, required=True, help='Email of Org Member who currently owns the documents') + parser.add_argument( + '--new_owner', type=str, required=True, help='Email of Org Member who you want to transfer the documents to') parser.add_argument( - '--new_owner', type=str, required=True, help='Org Member who you want to transfer the documents to') + '--new_folder', type=str, required=False, help='Optional folder to place the files in') args = parser.parse_args() client = SigmaClient(args.env, args.cloud, @@ -159,11 +168,11 @@ def main(): raise SystemExit("Script aborted") # filter to only docs they own - filtered_arr = [p for p in member_files if p['ownerId'] == curr_owner_id] + filtered_arr = [file for file in member_files if file['ownerId'] == curr_owner_id] # loop through and reassign ownership - for d in filtered_arr: + for ownedDoc in filtered_arr: try: - update_member_response = update_file(client, new_owner_id, d['id']) + update_member_response = update_file(client, new_owner_id, ownedDoc['id'], args.new_folder) except Exception as e: print(f"{e}") raise SystemExit("Script aborted") From c74b7f682f1f8d4406478a660d04c4ad542ee9cf Mon Sep 17 00:00:00 2001 From: Brett Bedevian Date: Tue, 22 Oct 2024 10:50:51 -0400 Subject: [PATCH 4/4] updates cont --- bulk_reassign_docs.py | 45 +++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/bulk_reassign_docs.py b/bulk_reassign_docs.py index 3defb42..cf502e1 100755 --- a/bulk_reassign_docs.py +++ b/bulk_reassign_docs.py @@ -53,12 +53,12 @@ def get_member_files(client, user_id): """ data = [] moreResults = True - nextPage = '' + next_page = '' while moreResults: try: # currently looks for workbooks and datasets but can be changed as needed response = client.get( - f"v2/members/{user_id}/files?typeFilters=workbook&typeFilters=dataset&limit=1000{nextPage}" + f"v2/members/{user_id}/files?typeFilters=workbook&typeFilters=dataset&limit=1000{next_page}" ) response.raise_for_status() @@ -80,24 +80,49 @@ def get_member_files(client, user_id): if resp['nextPage'] is None: moreResults = False else: - pageID = str(resp['nextPage']) - nextPage = f'&page={pageID}' + page_id = str(resp['nextPage']) + next_page = f'&page={page_id}' return data -def update_file(client, user_id, file_id, folderID): +def update_file(client, user_id, file_id, folder_id): """ Update file :access_token: Generated access token - :userId: ID of the new owner + :user_id: ID of the new owner :file_id: File to transfer ownership of :returns: Response JSON """ updateFileBody={"ownerId": user_id} - if folderID: - updateFileBody["parentId"] = folderID + if folder_id: + # we need to make sure this person owns the folder + try: + response = client.get( + f"v2/files/{folder_id}", + ) + response.raise_for_status() + # HTTP and other errors are handled generally by raising them as exceptions to be surfaced in downstream code + except requests.exceptions.HTTPError as errh: + raise Exception(errh.response.status_code, + f"API message: {errh.response.json()['message']}") + except requests.exceptions.ConnectionError as errc: + raise Exception( + f"Connection Error: {errc}, API response: {errc.response.text}") + except requests.exceptions.Timeout as errt: + raise Exception( + f"Timeout Error: {errt}, API response: {errt.response.text}") + except requests.exceptions.RequestException as err: + raise Exception( + f"Other Error: {err}, API response: {err.response.text}") + else: + get_folder_response = response.json() + if get_folder_response['type'] == "folder": + updateFileBody["parentId"] = folder_id + else: + print("Provided folder_id not found") + raise SystemExit("Script aborted") try: response = client.patch( f"v2/files/{file_id}", @@ -170,9 +195,9 @@ def main(): # filter to only docs they own filtered_arr = [file for file in member_files if file['ownerId'] == curr_owner_id] # loop through and reassign ownership - for ownedDoc in filtered_arr: + for owned_doc in filtered_arr: try: - update_member_response = update_file(client, new_owner_id, ownedDoc['id'], args.new_folder) + update_file(client, new_owner_id, owned_doc['id'], args.new_folder) except Exception as e: print(f"{e}") raise SystemExit("Script aborted")