From 6d6a3bffd5d5ba79b69ce9cb760d330442349657 Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Tue, 1 Aug 2023 23:29:09 +0100 Subject: [PATCH 1/7] Add to help customers invalidate derived versions of transformations --- cloudinary_cli/modules/__init__.py | 2 + .../delete_all_derived_by_transformation.py | 106 ++++++++++++++++++ cloudinary_cli/utils/api_utils.py | 23 ++++ 3 files changed, 131 insertions(+) create mode 100644 cloudinary_cli/modules/delete_all_derived_by_transformation.py diff --git a/cloudinary_cli/modules/__init__.py b/cloudinary_cli/modules/__init__.py index f1a3086..f75884c 100644 --- a/cloudinary_cli/modules/__init__.py +++ b/cloudinary_cli/modules/__init__.py @@ -2,10 +2,12 @@ from .migrate import migrate from .sync import sync from .upload_dir import upload_dir +from .delete_all_derived_by_transformation import delete_all_derived_by_transformation commands = [ upload_dir, make, migrate, sync, + delete_all_derived_by_transformation ] diff --git a/cloudinary_cli/modules/delete_all_derived_by_transformation.py b/cloudinary_cli/modules/delete_all_derived_by_transformation.py new file mode 100644 index 0000000..6c125b3 --- /dev/null +++ b/cloudinary_cli/modules/delete_all_derived_by_transformation.py @@ -0,0 +1,106 @@ +from click import command, argument, option +from cloudinary_cli.utils.utils import print_help_and_exit +from cloudinary import uploader as upload_api +from cloudinary_cli.utils.api_utils import handle_module_command, handle_command +from cloudinary import api +from cloudinary_cli.utils.utils import confirm_action +from cloudinary_cli.defaults import logger +import logging +from datetime import datetime + +DEFAULT_MAX_RESULTS = 500 + +curr_dt = datetime.today().strftime('%Y-%m-%d') +filename = f'delete_derived_public_ids_{curr_dt}.log' +logging.basicConfig(filename=filename, + level=logging.INFO, + format='%(asctime)s %(levelname)s - %(message)s') + + +@command("delete_all_derived_by_transformation", + short_help="""Delete all derived of a transformation.""", + help=""" +\b +Delete all derived versions of a named transformations. +Format: cld delete_all_derived_by_transformation +e.g. cld delete_all_derived_by_transformation t_named -A -ea -enu http://mywebhook.com +""") +@argument("trans_str") +@option("-enu", "--eager_notification_url", help="Webhook notification URL.") +@option("-ea", "--eager_async", is_flag=True, default=False, + help="Generate asynchronously.") +@option("-A", "--auto_paginate", is_flag=True, default=False, + help="Will auto paginate Admin API calls.") +@option("-F", "--force", is_flag=True, + help="Skip confirmation when running --auto-paginate/-A.") +@option("-fi", "--force_initial", is_flag=True, + help="Skip initial confirmation when running this module.") +@option("-n", "--max_results", nargs=1, default=10, + help="""The maximum number of derived results to return. + Default: 10, maximum: 500.""") +def delete_all_derived_by_transformation(trans_str, eager_notification_url, + eager_async, auto_paginate, force, + force_initial, max_results): + if not any(trans_str): + print_help_and_exit() + + if not force_initial: + if not confirm_action( + f"Running this module will explicity " + f"re-generate all the derived versions " + f"which will cause an increase in your trasnformation costs " + f"based on the number of derived re-generated.\n" + f"Continue? (y/N)"): + logger.info("Stopping.") + exit() + else: + logger.info("Continuing. You may use the -fi " + "flag to skip initial confirmation.") + + if auto_paginate: + max_results = DEFAULT_MAX_RESULTS + + params = ('transformation', trans_str, f'max_results={max_results}') + res = handle_module_command(params, (), (), api_instance=api, + api_name="admin", auto_paginate=auto_paginate, + force=force) + + if res: + eager_trans = trans_str + is_named = res.get('named') + if is_named: + if not eager_trans.startswith('t_'): + eager_trans = 't_' + eager_trans + logger.info(f"Regenerating {len(res.get('derived'))} derived versions " + f"with eager_async={eager_async}...\n" + f"Output is saved in {filename}") + for derived in res.get('derived'): + public_id = derived.get('public_id') + delivery_type = derived.get('type') + res_type = derived.get('resource_type') + params = ('explicit', public_id, f'type={delivery_type}', + f'resource_type={res_type}', f'eager={eager_trans}', + f'eager_async={eager_async}', + f'eager_notification_url={eager_notification_url}', + 'overwrite=True', 'invalidate=True') + + try: + exp_res = handle_command(params, (), (), module=upload_api, + module_name="delete_all_derived_by_transformation") + except Exception as e: + error_data = (f'public_id={public_id}, type={delivery_type}, ' + f'resource_type={res_type}: {e}') + logging.error(error_data) + continue + + if eager_async: + msg = f'Processing {exp_res.get("eager")[0].get("secure_url")}' + else: + msg = f'Generated {exp_res.get("eager")[0].get("secure_url")}' + + logging.info(msg) + logger.info("Please contact support if you do not see " + "updated derived versions after running this. " + "Do not re-run or it will cost more.") + + return True diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 5014711..473ba76 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -145,6 +145,29 @@ def handle_command( return call_api(func, args, kwargs) +def handle_module_command(params, optional_parameter, + optional_parameter_parsed, api_instance, + api_name, auto_paginate, force): + try: + func, args, kwargs = get_command_params(params, optional_parameter, + optional_parameter_parsed, + api_instance, api_name) + except Exception as e: + log_exception(e) + return False + + try: + res = call_api(func, args, kwargs) + except Exception: + return False + + if auto_paginate: + res = handle_auto_pagination(res, func, args, kwargs, + force=force, filter_fields=None) + + return res + + def handle_api_command( params, optional_parameter, From 5ffb4f0b838151dbef2cf1bb160123d103eedb37 Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Thu, 3 Aug 2023 13:52:06 +0100 Subject: [PATCH 2/7] Updaing script name from delete to regenerate --- cloudinary_cli/modules/__init__.py | 4 ++-- .../delete_all_derived_by_transformation.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cloudinary_cli/modules/__init__.py b/cloudinary_cli/modules/__init__.py index f75884c..4146fa7 100644 --- a/cloudinary_cli/modules/__init__.py +++ b/cloudinary_cli/modules/__init__.py @@ -2,12 +2,12 @@ from .migrate import migrate from .sync import sync from .upload_dir import upload_dir -from .delete_all_derived_by_transformation import delete_all_derived_by_transformation +from .regenerate_all_derived_by_transformation import regenerate_all_derived_by_transformation commands = [ upload_dir, make, migrate, sync, - delete_all_derived_by_transformation + regenerate_all_derived_by_transformation ] diff --git a/cloudinary_cli/modules/delete_all_derived_by_transformation.py b/cloudinary_cli/modules/delete_all_derived_by_transformation.py index 6c125b3..e7ed1c2 100644 --- a/cloudinary_cli/modules/delete_all_derived_by_transformation.py +++ b/cloudinary_cli/modules/delete_all_derived_by_transformation.py @@ -11,19 +11,19 @@ DEFAULT_MAX_RESULTS = 500 curr_dt = datetime.today().strftime('%Y-%m-%d') -filename = f'delete_derived_public_ids_{curr_dt}.log' +filename = f'regenerate_derived_public_ids_{curr_dt}.log' logging.basicConfig(filename=filename, level=logging.INFO, format='%(asctime)s %(levelname)s - %(message)s') -@command("delete_all_derived_by_transformation", - short_help="""Delete all derived of a transformation.""", +@command("regenerate_all_derived_by_transformation", + short_help="""Regenerate all derived of a transformation.""", help=""" \b -Delete all derived versions of a named transformations. -Format: cld delete_all_derived_by_transformation -e.g. cld delete_all_derived_by_transformation t_named -A -ea -enu http://mywebhook.com +Regenerate all derived versions of a named transformations. +Format: cld regenerate_all_derived_by_transformation +e.g. cld regenerate_all_derived_by_transformation t_named -A -ea -enu http://mywebhook.com """) @argument("trans_str") @option("-enu", "--eager_notification_url", help="Webhook notification URL.") @@ -38,7 +38,7 @@ @option("-n", "--max_results", nargs=1, default=10, help="""The maximum number of derived results to return. Default: 10, maximum: 500.""") -def delete_all_derived_by_transformation(trans_str, eager_notification_url, +def regenerate_all_derived_by_transformation(trans_str, eager_notification_url, eager_async, auto_paginate, force, force_initial, max_results): if not any(trans_str): @@ -48,7 +48,7 @@ def delete_all_derived_by_transformation(trans_str, eager_notification_url, if not confirm_action( f"Running this module will explicity " f"re-generate all the derived versions " - f"which will cause an increase in your trasnformation costs " + f"which will cause an increase in your transformation costs " f"based on the number of derived re-generated.\n" f"Continue? (y/N)"): logger.info("Stopping.") @@ -86,7 +86,7 @@ def delete_all_derived_by_transformation(trans_str, eager_notification_url, try: exp_res = handle_command(params, (), (), module=upload_api, - module_name="delete_all_derived_by_transformation") + module_name="regenerate_all_derived_by_transformation") except Exception as e: error_data = (f'public_id={public_id}, type={delivery_type}, ' f'resource_type={res_type}: {e}') From 59346988ea54ecd001606c2ec0a812eaec72dd3d Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Thu, 3 Aug 2023 13:54:08 +0100 Subject: [PATCH 3/7] Rename file to regenerate as well --- ...nsformation.py => regenerate_all_derived_by_transformation.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cloudinary_cli/modules/{delete_all_derived_by_transformation.py => regenerate_all_derived_by_transformation.py} (100%) diff --git a/cloudinary_cli/modules/delete_all_derived_by_transformation.py b/cloudinary_cli/modules/regenerate_all_derived_by_transformation.py similarity index 100% rename from cloudinary_cli/modules/delete_all_derived_by_transformation.py rename to cloudinary_cli/modules/regenerate_all_derived_by_transformation.py From 2407ced52b183ded20ed864704addc5023240db8 Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Tue, 5 Sep 2023 14:40:23 +0100 Subject: [PATCH 4/7] Adding changes per discussion --- cloudinary_cli/modules/__init__.py | 4 +- cloudinary_cli/modules/regen_derived.py | 102 +++++++++++++++++ ...egenerate_all_derived_by_transformation.py | 106 ------------------ cloudinary_cli/utils/api_utils.py | 48 ++++---- 4 files changed, 127 insertions(+), 133 deletions(-) create mode 100644 cloudinary_cli/modules/regen_derived.py delete mode 100644 cloudinary_cli/modules/regenerate_all_derived_by_transformation.py diff --git a/cloudinary_cli/modules/__init__.py b/cloudinary_cli/modules/__init__.py index 4146fa7..996eab9 100644 --- a/cloudinary_cli/modules/__init__.py +++ b/cloudinary_cli/modules/__init__.py @@ -2,12 +2,12 @@ from .migrate import migrate from .sync import sync from .upload_dir import upload_dir -from .regenerate_all_derived_by_transformation import regenerate_all_derived_by_transformation +from .regen_derived import regen_derived commands = [ upload_dir, make, migrate, sync, - regenerate_all_derived_by_transformation + regen_derived ] diff --git a/cloudinary_cli/modules/regen_derived.py b/cloudinary_cli/modules/regen_derived.py new file mode 100644 index 0000000..f5e3a7f --- /dev/null +++ b/cloudinary_cli/modules/regen_derived.py @@ -0,0 +1,102 @@ +from click import command, argument, option +from cloudinary_cli.utils.utils import print_help_and_exit +from cloudinary_cli.utils.api_utils import handle_api_command, regen_derived_version +from cloudinary import api +from cloudinary_cli.utils.utils import confirm_action, run_tasks_concurrently +from cloudinary_cli.defaults import logger + +DEFAULT_MAX_RESULTS = 500 + +@command("regen_derived", + short_help="""Regenerate all derived of a transformation.""", + help=""" +\b +Regenerate all derived versions of a named transformations. +Format: cld regen_dervied +e.g. cld regen_derived t_named -A -ea -enu http://mywebhook.com +""") +@argument("trans_str") +@option("-enu", "--eager_notification_url", help="Webhook notification URL.") +@option("-ea", "--eager_async", is_flag=True, default=False, + help="Generate asynchronously.") +@option("-A", "--auto_paginate", is_flag=True, default=False, + help="Will auto paginate Admin API calls.") +@option("-F", "--force", is_flag=True, + help="Skip confirmation when running --auto-paginate/-A.") +@option("-n", "--max_results", nargs=1, default=10, + help="""The maximum number of derived results to return. + Default: 10, maximum: 500.""") +@option("-w", "--concurrent_workers", type=int, default=30, + help="Specify the number of concurrent network threads.") +def regen_derived(trans_str, eager_notification_url, + eager_async, auto_paginate, force, + max_results, concurrent_workers): + + if not any(trans_str): + print_help_and_exit() + + if not force: + if not confirm_action( + f"Running this module will explicity " + f"re-generate all the related derived versions " + f"which will cause an increase in your transformation costs " + f"based on the number of derived re-generated.\n" + f"If running in auto_paginate (-A) mode, " + f"multiple Admin API (rate-limited) calls will be used as well.\n" + f"Continue? (y/N)"): + logger.info("Stopping.") + exit() + else: + logger.info("Continuing. You may use the -F " + "flag to skip confirmation.") + + if auto_paginate: + max_results = DEFAULT_MAX_RESULTS + force = True + + params = ('transformation', trans_str, f'max_results={max_results}') + res = handle_api_command(params, (), (), None, None, None, doc_url="", + api_instance=api, api_name="admin", + auto_paginate=auto_paginate, force=force, + return_data=True) + derived_resources = res.get('derived') + if not derived_resources: + logger.info("No derived resources using this transformation.") + exit() + + is_named = res.get('named') + if is_named: + eager_trans = normalise_trans_name(trans_str) + + progress_msg = f"Regenerating {len(derived_resources)} derived version(s)" + if eager_async: + logger.info(f"{progress_msg} " + f"with eager_async={eager_async}...") + else: + logger.info(f"{progress_msg}...") + + regen_conc_list = [] + for derived in derived_resources: + public_id = derived.get('public_id') + delivery_type = derived.get('type') + res_type = derived.get('resource_type') + options = {"type": delivery_type, "resource_type": res_type, + "eager": eager_trans, "eager_async": eager_async, + "eager_notification_url": eager_notification_url, + "overwrite": True, "invalidate": True} + regen_conc_list.append((public_id, options)) + + run_tasks_concurrently(regen_derived_version, regen_conc_list, + concurrent_workers) + + logger.info("Regen complete. It may take up to 10 mins " + "to see the changes. Please contact support " + "if you still see the old media.") + return True + + +def normalise_trans_name(trans_name): + normalised_trans_name = trans_name + if not normalised_trans_name.startswith('t_'): + normalised_trans_name = 't_' + normalised_trans_name + return normalised_trans_name diff --git a/cloudinary_cli/modules/regenerate_all_derived_by_transformation.py b/cloudinary_cli/modules/regenerate_all_derived_by_transformation.py deleted file mode 100644 index e7ed1c2..0000000 --- a/cloudinary_cli/modules/regenerate_all_derived_by_transformation.py +++ /dev/null @@ -1,106 +0,0 @@ -from click import command, argument, option -from cloudinary_cli.utils.utils import print_help_and_exit -from cloudinary import uploader as upload_api -from cloudinary_cli.utils.api_utils import handle_module_command, handle_command -from cloudinary import api -from cloudinary_cli.utils.utils import confirm_action -from cloudinary_cli.defaults import logger -import logging -from datetime import datetime - -DEFAULT_MAX_RESULTS = 500 - -curr_dt = datetime.today().strftime('%Y-%m-%d') -filename = f'regenerate_derived_public_ids_{curr_dt}.log' -logging.basicConfig(filename=filename, - level=logging.INFO, - format='%(asctime)s %(levelname)s - %(message)s') - - -@command("regenerate_all_derived_by_transformation", - short_help="""Regenerate all derived of a transformation.""", - help=""" -\b -Regenerate all derived versions of a named transformations. -Format: cld regenerate_all_derived_by_transformation -e.g. cld regenerate_all_derived_by_transformation t_named -A -ea -enu http://mywebhook.com -""") -@argument("trans_str") -@option("-enu", "--eager_notification_url", help="Webhook notification URL.") -@option("-ea", "--eager_async", is_flag=True, default=False, - help="Generate asynchronously.") -@option("-A", "--auto_paginate", is_flag=True, default=False, - help="Will auto paginate Admin API calls.") -@option("-F", "--force", is_flag=True, - help="Skip confirmation when running --auto-paginate/-A.") -@option("-fi", "--force_initial", is_flag=True, - help="Skip initial confirmation when running this module.") -@option("-n", "--max_results", nargs=1, default=10, - help="""The maximum number of derived results to return. - Default: 10, maximum: 500.""") -def regenerate_all_derived_by_transformation(trans_str, eager_notification_url, - eager_async, auto_paginate, force, - force_initial, max_results): - if not any(trans_str): - print_help_and_exit() - - if not force_initial: - if not confirm_action( - f"Running this module will explicity " - f"re-generate all the derived versions " - f"which will cause an increase in your transformation costs " - f"based on the number of derived re-generated.\n" - f"Continue? (y/N)"): - logger.info("Stopping.") - exit() - else: - logger.info("Continuing. You may use the -fi " - "flag to skip initial confirmation.") - - if auto_paginate: - max_results = DEFAULT_MAX_RESULTS - - params = ('transformation', trans_str, f'max_results={max_results}') - res = handle_module_command(params, (), (), api_instance=api, - api_name="admin", auto_paginate=auto_paginate, - force=force) - - if res: - eager_trans = trans_str - is_named = res.get('named') - if is_named: - if not eager_trans.startswith('t_'): - eager_trans = 't_' + eager_trans - logger.info(f"Regenerating {len(res.get('derived'))} derived versions " - f"with eager_async={eager_async}...\n" - f"Output is saved in {filename}") - for derived in res.get('derived'): - public_id = derived.get('public_id') - delivery_type = derived.get('type') - res_type = derived.get('resource_type') - params = ('explicit', public_id, f'type={delivery_type}', - f'resource_type={res_type}', f'eager={eager_trans}', - f'eager_async={eager_async}', - f'eager_notification_url={eager_notification_url}', - 'overwrite=True', 'invalidate=True') - - try: - exp_res = handle_command(params, (), (), module=upload_api, - module_name="regenerate_all_derived_by_transformation") - except Exception as e: - error_data = (f'public_id={public_id}, type={delivery_type}, ' - f'resource_type={res_type}: {e}') - logging.error(error_data) - continue - - if eager_async: - msg = f'Processing {exp_res.get("eager")[0].get("secure_url")}' - else: - msg = f'Generated {exp_res.get("eager")[0].get("secure_url")}' - - logging.info(msg) - logger.info("Please contact support if you do not see " - "updated derived versions after running this. " - "Do not re-run or it will cost more.") - - return True diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 473ba76..96382a0 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -48,6 +48,22 @@ def query_cld_folder(folder): return files +def regen_derived_version(public_id, options): + try: + exp_res = uploader.explicit(public_id, **options) + derived_url = f'{exp_res.get("eager")[0].get("secure_url")}' + if (options.get('eager_async')): + msg = f'Processing {derived_url}' + else: + msg = f'Regenerated {derived_url}' + logger.info(style(msg, fg="green")) + except Exception as e: + error_msg = (f"Failed to regenerate {public_id} of type: " + f"{options.get('type')} and resource_type: " + f"{options.get('resource_type')}") + log_exception(e, error_msg) + + def upload_file(file_path, options, uploaded=None, failed=None): uploaded = uploaded if uploaded is not None else {} failed = failed if failed is not None else {} @@ -145,29 +161,6 @@ def handle_command( return call_api(func, args, kwargs) -def handle_module_command(params, optional_parameter, - optional_parameter_parsed, api_instance, - api_name, auto_paginate, force): - try: - func, args, kwargs = get_command_params(params, optional_parameter, - optional_parameter_parsed, - api_instance, api_name) - except Exception as e: - log_exception(e) - return False - - try: - res = call_api(func, args, kwargs) - except Exception: - return False - - if auto_paginate: - res = handle_auto_pagination(res, func, args, kwargs, - force=force, filter_fields=None) - - return res - - def handle_api_command( params, optional_parameter, @@ -180,10 +173,12 @@ def handle_api_command( api_name, auto_paginate=False, force=False, - filter_fields=None): + filter_fields=None, + return_data=False): """ Used by Admin and Upload API commands """ + if doc: return launch(doc_url) @@ -208,7 +203,10 @@ def handle_api_command( if auto_paginate: res = handle_auto_pagination(res, func, args, kwargs, force, filter_fields) - print_json(res) + if return_data: + return res + else: + print_json(res) if save: write_json_to_file(res, save) From be284471f00edec7b9075e6fa8effa2537c83eb9 Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Tue, 5 Sep 2023 14:54:26 +0100 Subject: [PATCH 5/7] Adding changes per discussion --- cloudinary_cli/modules/regen_derived.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cloudinary_cli/modules/regen_derived.py b/cloudinary_cli/modules/regen_derived.py index f5e3a7f..9c2b7a8 100644 --- a/cloudinary_cli/modules/regen_derived.py +++ b/cloudinary_cli/modules/regen_derived.py @@ -55,16 +55,17 @@ def regen_derived(trans_str, eager_notification_url, force = True params = ('transformation', trans_str, f'max_results={max_results}') - res = handle_api_command(params, (), (), None, None, None, doc_url="", - api_instance=api, api_name="admin", - auto_paginate=auto_paginate, force=force, - return_data=True) - derived_resources = res.get('derived') + trans_details = handle_api_command(params, (), (), None, None, None, + doc_url="", api_instance=api, + api_name="admin", + auto_paginate=auto_paginate, + force=force, return_data=True) + derived_resources = trans_details.get('derived') if not derived_resources: logger.info("No derived resources using this transformation.") exit() - is_named = res.get('named') + is_named = trans_details.get('named') if is_named: eager_trans = normalise_trans_name(trans_str) From 60b8468edb0e54de16e66a170e1d6d5df4600e35 Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Wed, 13 Sep 2023 13:43:16 +0100 Subject: [PATCH 6/7] Adding changes per code review --- cloudinary_cli/modules/regen_derived.py | 30 ++++++++++--------------- cloudinary_cli/utils/api_utils.py | 17 ++++++++------ 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/cloudinary_cli/modules/regen_derived.py b/cloudinary_cli/modules/regen_derived.py index 9c2b7a8..fa9f27f 100644 --- a/cloudinary_cli/modules/regen_derived.py +++ b/cloudinary_cli/modules/regen_derived.py @@ -7,6 +7,7 @@ DEFAULT_MAX_RESULTS = 500 + @command("regen_derived", short_help="""Regenerate all derived of a transformation.""", help=""" @@ -22,7 +23,7 @@ @option("-A", "--auto_paginate", is_flag=True, default=False, help="Will auto paginate Admin API calls.") @option("-F", "--force", is_flag=True, - help="Skip confirmation when running --auto-paginate/-A.") + help="Skip initial and auto_paginate confirmation.") @option("-n", "--max_results", nargs=1, default=10, help="""The maximum number of derived results to return. Default: 10, maximum: 500.""") @@ -66,38 +67,31 @@ def regen_derived(trans_str, eager_notification_url, exit() is_named = trans_details.get('named') - if is_named: - eager_trans = normalise_trans_name(trans_str) + eager_trans = normalise_trans_name(trans_str) if is_named else trans_str progress_msg = f"Regenerating {len(derived_resources)} derived version(s)" if eager_async: - logger.info(f"{progress_msg} " - f"with eager_async={eager_async}...") - else: - logger.info(f"{progress_msg}...") + progress_msg += f" with eager_async={eager_async}" + logger.info(f"{progress_msg}...") regen_conc_list = [] for derived in derived_resources: public_id = derived.get('public_id') delivery_type = derived.get('type') res_type = derived.get('resource_type') - options = {"type": delivery_type, "resource_type": res_type, - "eager": eager_trans, "eager_async": eager_async, - "eager_notification_url": eager_notification_url, - "overwrite": True, "invalidate": True} - regen_conc_list.append((public_id, options)) + regen_conc_list.append((public_id, delivery_type, res_type, + eager_trans, eager_async, + eager_notification_url)) run_tasks_concurrently(regen_derived_version, regen_conc_list, concurrent_workers) - - logger.info("Regen complete. It may take up to 10 mins " + complete_msg = ('All derived sent for processing' + if eager_async else 'Regen complete') + logger.info(f"{complete_msg}. It may take up to 10 mins " "to see the changes. Please contact support " "if you still see the old media.") return True def normalise_trans_name(trans_name): - normalised_trans_name = trans_name - if not normalised_trans_name.startswith('t_'): - normalised_trans_name = 't_' + normalised_trans_name - return normalised_trans_name + return trans_name if trans_name.startswith('t_') else 't_' + trans_name diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 96382a0..7d25176 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -48,14 +48,17 @@ def query_cld_folder(folder): return files -def regen_derived_version(public_id, options): +def regen_derived_version(public_id, delivery_type, res_type, + eager_trans, eager_async, + eager_notification_url): + options = {"type": delivery_type, "resource_type": res_type, + "eager": eager_trans, "eager_async": eager_async, + "eager_notification_url": eager_notification_url, + "overwrite": True, "invalidate": True} try: exp_res = uploader.explicit(public_id, **options) derived_url = f'{exp_res.get("eager")[0].get("secure_url")}' - if (options.get('eager_async')): - msg = f'Processing {derived_url}' - else: - msg = f'Regenerated {derived_url}' + msg = ('Processing' if options.get('eager_async') else 'Regenerated') + f' {derived_url}' logger.info(style(msg, fg="green")) except Exception as e: error_msg = (f"Failed to regenerate {public_id} of type: " @@ -205,8 +208,8 @@ def handle_api_command( if return_data: return res - else: - print_json(res) + + print_json(res) if save: write_json_to_file(res, save) From 4b441d932d7e286a0a31958d22e9cafa42ceb0ff Mon Sep 17 00:00:00 2001 From: Thomas Gurung Date: Tue, 19 Sep 2023 12:14:27 +0100 Subject: [PATCH 7/7] Updating doc strings --- cloudinary_cli/modules/regen_derived.py | 30 +++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/cloudinary_cli/modules/regen_derived.py b/cloudinary_cli/modules/regen_derived.py index fa9f27f..a0b948b 100644 --- a/cloudinary_cli/modules/regen_derived.py +++ b/cloudinary_cli/modules/regen_derived.py @@ -9,11 +9,13 @@ @command("regen_derived", - short_help="""Regenerate all derived of a transformation.""", + short_help="""Regenerate all derived assets pertaining \ + to a named transformation, or transformation string.""", help=""" \b -Regenerate all derived versions of a named transformations. -Format: cld regen_dervied +Regenerate all derived assets pertaining to a specific named transformation, or transformation string. +Use this after updating a named transformation to invalidate and repopulate the cache with up-to-date versions of the assets. +Format: cld regen_derived e.g. cld regen_derived t_named -A -ea -enu http://mywebhook.com """) @argument("trans_str") @@ -21,11 +23,11 @@ @option("-ea", "--eager_async", is_flag=True, default=False, help="Generate asynchronously.") @option("-A", "--auto_paginate", is_flag=True, default=False, - help="Will auto paginate Admin API calls.") + help="Auto-paginate Admin API calls.") @option("-F", "--force", is_flag=True, - help="Skip initial and auto_paginate confirmation.") + help="Skip initial and auto-paginate confirmation.") @option("-n", "--max_results", nargs=1, default=10, - help="""The maximum number of derived results to return. + help="""The maximum number of results to return. Default: 10, maximum: 500.""") @option("-w", "--concurrent_workers", type=int, default=30, help="Specify the number of concurrent network threads.") @@ -39,11 +41,11 @@ def regen_derived(trans_str, eager_notification_url, if not force: if not confirm_action( f"Running this module will explicity " - f"re-generate all the related derived versions " + f"re-generate all the related derived assets " f"which will cause an increase in your transformation costs " - f"based on the number of derived re-generated.\n" - f"If running in auto_paginate (-A) mode, " - f"multiple Admin API (rate-limited) calls will be used as well.\n" + f"based on the number of derived assets re-generated.\n" + f"If running in auto-paginate (-A) mode, " + f"multiple Admin API (rate-limited) calls will be made.\n" f"Continue? (y/N)"): logger.info("Stopping.") exit() @@ -63,13 +65,13 @@ def regen_derived(trans_str, eager_notification_url, force=force, return_data=True) derived_resources = trans_details.get('derived') if not derived_resources: - logger.info("No derived resources using this transformation.") + logger.info("No derived assets are using this transformation.") exit() is_named = trans_details.get('named') eager_trans = normalise_trans_name(trans_str) if is_named else trans_str - progress_msg = f"Regenerating {len(derived_resources)} derived version(s)" + progress_msg = f"Regenerating {len(derived_resources)} derived asset(s)" if eager_async: progress_msg += f" with eager_async={eager_async}" logger.info(f"{progress_msg}...") @@ -85,8 +87,8 @@ def regen_derived(trans_str, eager_notification_url, run_tasks_concurrently(regen_derived_version, regen_conc_list, concurrent_workers) - complete_msg = ('All derived sent for processing' - if eager_async else 'Regen complete') + complete_msg = ('Regeneration in progress' + if eager_async else 'Regeneration complete') logger.info(f"{complete_msg}. It may take up to 10 mins " "to see the changes. Please contact support " "if you still see the old media.")