Add support for dynamic folders mode in sync command

cloudinary_cli/modules/sync.py

@@ -3,14 +3,15 @@
 from itertools import groupby
 from os import path, remove
 
-from click import command, argument, option, style, UsageError
+from click import command, argument, option, style, UsageError, Choice
 from cloudinary import api
 
-from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file
+from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file, get_folder_mode
 from cloudinary_cli.utils.file_utils import walk_dir, delete_empty_dirs, get_destination_folder, \
     normalize_file_extension, posix_rel_path
 from cloudinary_cli.utils.json_utils import print_json, read_json_from_file, write_json_to_file
-from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action, invert_dict, chunker
+from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action, invert_dict, chunker, \
+    group_params, parse_option_value
 
 _DEFAULT_DELETION_BATCH_SIZE = 30
 _DEFAULT_CONCURRENT_WORKERS = 30
@@ -32,13 +33,18 @@
 @option("-K", "--keep-unique", is_flag=True, help="Keep unique files in the destination folder.")
 @option("-D", "--deletion-batch-size", type=int, default=_DEFAULT_DELETION_BATCH_SIZE,
         help="Specify the batch size for deleting remote assets.")
+@option("-fm", "--folder-mode", type=Choice(['fixed', 'dynamic'], case_sensitive=False),
+        help="Specify folder mode explicitly. By default uses cloud mode configured in your cloud.")
+@option("-o", "--optional_parameter", multiple=True, nargs=2, help="Pass optional parameters as raw strings.")
+@option("-O", "--optional_parameter_parsed", multiple=True, nargs=2,
+        help="Pass optional parameters as interpreted strings.")
 def sync(local_folder, cloudinary_folder, push, pull, include_hidden, concurrent_workers, force, keep_unique,
-         deletion_batch_size):
+         deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed):
     if push == pull:
         raise UsageError("Please use either the '--push' OR '--pull' options")
 
     sync_dir = SyncDir(local_folder, cloudinary_folder, include_hidden, concurrent_workers, force, keep_unique,
-                       deletion_batch_size)
+                       deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed)
 
     result = True
     if push:
@@ -53,7 +59,7 @@ def sync(local_folder, cloudinary_folder, push, pull, include_hidden, concurrent
 
 class SyncDir:
     def __init__(self, local_dir, remote_dir, include_hidden, concurrent_workers, force, keep_deleted,
-                 deletion_batch_size):
+                 deletion_batch_size, folder_mode, optional_parameter, optional_parameter_parsed):
         self.local_dir = local_dir
         self.remote_dir = remote_dir.strip('/')
         self.user_friendly_remote_dir = self.remote_dir if self.remote_dir else '/'
@@ -63,15 +69,21 @@ def __init__(self, local_dir, remote_dir, include_hidden, concurrent_workers, fo
         self.keep_unique = keep_deleted
         self.deletion_batch_size = deletion_batch_size
 
+        self.folder_mode = folder_mode or get_folder_mode()
+
+        self.optional_parameter = optional_parameter
+        self.optional_parameter_parsed = optional_parameter_parsed
+
         self.sync_meta_file = path.join(self.local_dir, _SYNC_META_FILE)
 
         self.verbose = logger.getEffectiveLevel() < logging.INFO
 
         self.local_files = walk_dir(path.abspath(self.local_dir), include_hidden)
         logger.info(f"Found {len(self.local_files)} items in local folder '{local_dir}'")
 
-        self.remote_files = query_cld_folder(self.remote_dir)
-        logger.info(f"Found {len(self.remote_files)} items in Cloudinary folder '{self.user_friendly_remote_dir}'")
+        self.remote_files = query_cld_folder(self.remote_dir, self.folder_mode)
+        logger.info(f"Found {len(self.remote_files)} items in Cloudinary folder '{self.user_friendly_remote_dir}' "
+                    f"({self.folder_mode} folder mode)")
 
         local_file_names = self.local_files.keys()
         remote_file_names = self.remote_files.keys()
@@ -122,20 +134,16 @@ def push(self):
 
         logger.info(f"Uploading {len(files_to_push)} items to Cloudinary folder '{self.user_friendly_remote_dir}'")
 
-        options = {
-            'use_filename': True,
-            'unique_filename': False,
-            'invalidate': True,
-            'resource_type': 'auto'
-        }
+        options = self.get_upload_options()
+
         upload_results = {}
         upload_errors = {}
         uploads = []
         for file in files_to_push:
-            folder = get_destination_folder(self.remote_dir, file)
+            folder_options = self.get_destination_folder_options(file)
 
             uploads.append(
-                (self.local_files[file]['path'], {**options, 'folder': folder}, upload_results, upload_errors))
+                (self.local_files[file]['path'], {**options, **folder_options}, upload_results, upload_errors))
 
         try:
             run_tasks_concurrently(upload_file, uploads, self.concurrent_workers)
@@ -146,6 +154,43 @@ def push(self):
         if upload_errors:
             raise Exception("Sync did not finish successfully")
 
+    def get_destination_folder_options(self, file):
+        destination_folder = get_destination_folder(self.remote_dir, file)
+
+        if self.folder_mode == "dynamic":
+            return {"asset_folder": destination_folder}
+
+        return {"folder": destination_folder}
+
+    def get_upload_options(self):
+        options = {
+            'resource_type': 'auto'
+        }
+
+        if self.folder_mode == 'fixed':
+            options = {
+                **options,
+                'use_filename': True,
+                'unique_filename': False,
+                'invalidate': True,
+            }
+
+        if self.folder_mode == 'dynamic':
+            options = {
+                **options,
+                'use_filename_as_display_name': True,
+            }
+
+        options = {
+            **options,
+            **group_params(
+                self.optional_parameter,
+                ((k, parse_option_value(v)) for k, v in self.optional_parameter_parsed)
+            ),
+        }
+
+        return options
+
     def pull(self):
         """
         Pulls changes from the Cloudinary folder to the local folder.
@@ -186,7 +231,8 @@ def _print_sync_status(self, success, errors):
 
     def _save_sync_meta_file(self, upload_results):
         diverse_filenames = {}
-        for local_path, remote_path in upload_results.items():
+        for local_path, remote_res in upload_results.items():
+            remote_path = remote_res["display_path"] if self.folder_mode == "dynamic" else remote_res["path"]
             local = normalize_file_extension(posix_rel_path(local_path, self.local_dir))
             remote = normalize_file_extension(posix_rel_path(remote_path, self.remote_dir))
             if local != remote:
@@ -254,7 +300,10 @@ def _get_out_of_sync_file_names(self, common_file_names):
                 out_of_sync_file_names.add(f)
                 continue
             logger.debug(f"'{f}' is in sync" +
-                         (f" with '{self.diverse_file_names[f]}'" if f in self.diverse_file_names else ""))
+                         (f" with '{self.diverse_file_names[f]}'" if f in self.diverse_file_names else "") +
+                         (f". Public ID: {self.recovered_remote_files[f]['public_id']}"
+                          if self.folder_mode == "dynamic" else "")
+                         )
 
         return out_of_sync_file_names
 

cloudinary_cli/utils/api_utils.py

@@ -3,7 +3,7 @@
 
 import requests
 from click import style, launch
-from cloudinary import Search, uploader
+from cloudinary import Search, uploader, api
 from cloudinary.utils import cloudinary_url
 
 from cloudinary_cli.defaults import logger
@@ -18,7 +18,7 @@
 _cursor_fields = {"resource": "derived_next_cursor"}
 
 
-def query_cld_folder(folder):
+def query_cld_folder(folder, folder_mode):
     files = {}
 
     folder = folder.strip('/')  # omit redundant leading slash and duplicate trailing slashes in query
@@ -31,15 +31,21 @@ def query_cld_folder(folder):
         res = expression.execute()
 
         for asset in res['resources']:
-            rel_path = posix_rel_path(asset_source(asset), folder)
-            files[normalize_file_extension(rel_path)] = {
+            rel_path = _relative_path(asset, folder)
+            rel_display_path = _relative_display_path(asset, folder)
+            path_key = rel_display_path if folder_mode == "dynamic" else rel_path
+            files[normalize_file_extension(path_key)] = {
                 "type": asset['type'],
                 "resource_type": asset['resource_type'],
                 "public_id": asset['public_id'],
                 "format": asset['format'],
                 "etag": asset.get('etag', '0'),
                 "relative_path": rel_path,  # save for inner use
                 "access_mode": asset.get('access_mode', 'public'),
+                # dynamic folder mode fields
+                "asset_folder": asset.get('asset_folder'),
+                "display_name": asset.get('display_name'),
+                "relative_display_path": rel_display_path
             }
         # use := when switch to python 3.8
         next_cursor = res.get('next_cursor')
@@ -48,6 +54,28 @@ def query_cld_folder(folder):
     return files
 
 
+def _display_path(asset):
+    if asset.get("display_name") is None:
+        return ""
+
+    return "/".join([asset["asset_folder"], ".".join([asset["display_name"], asset["format"]])])
+
+
+def _relative_display_path(asset, folder):
+    if asset.get("display_name") is None:
+        return ""
+
+    return posix_rel_path(_display_path(asset), folder)
+
+
+def _relative_path(asset, folder):
+    source = asset_source(asset)
+    if not source.startswith(folder):
+        return source
+
+    return posix_rel_path(asset_source(asset), folder)
+
+
 def regen_derived_version(public_id, delivery_type, res_type,
                           eager_trans, eager_async,
                           eager_notification_url):
@@ -78,10 +106,13 @@ def upload_file(file_path, options, uploaded=None, failed=None):
         if size > 20000000:
             upload_func = uploader.upload_large
         result = upload_func(file_path, **options)
-        logger.info(style(f"Successfully uploaded {file_path} as {result['public_id']}", fg="green"))
+        disp_path = _display_path(result)
+        disp_str = f"as {result['public_id']}" if not disp_path \
+            else f"as {disp_path} with public_id: {result['public_id']}"
+        logger.info(style(f"Successfully uploaded {file_path} {disp_str}", fg="green"))
         if verbose:
             print_json(result)
-        uploaded[file_path] = asset_source(result)
+        uploaded[file_path] = {"path": asset_source(result), "display_path": disp_path}
     except Exception as e:
         log_exception(e, f"Failed uploading {file_path}")
         failed[file_path] = str(e)
@@ -134,12 +165,29 @@ def asset_source(asset_details):
     :return:
     """
     base_name = asset_details['public_id']
+
     if asset_details['resource_type'] == 'raw' or asset_details['type'] == 'fetch':
         return base_name
 
     return base_name + '.' + asset_details['format']
 
 
+def get_folder_mode():
+    """
+    Returns folder mode of the cloud.
+
+    :return: String representing folder mode. Can be "fixed" or "dynamic".
+    """
+    try:
+        config_res = api.config(settings="true")
+        mode = config_res["settings"]["folder_mode"]
+    except Exception as e:
+        log_exception(e, f"Failed getting cloud configuration")
+        raise
+
+    return mode
+
+
 def call_api(func, args, kwargs):
     try:
         return func(*args, **kwargs)