Skip to content

Commit

Permalink
split progress in two commands
Browse files Browse the repository at this point in the history
  • Loading branch information
@plaharanne
plaharanne committed Oct 17, 2023
1 parent 51a265b commit 764b760
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 42 deletions.
67 changes: 44 additions & 23 deletions croud/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@
export_jobs_create,
export_jobs_delete,
export_jobs_list,
import_job_progress,
import_job_progress_files,
import_job_progress_summary,
import_jobs_create_from_file,
import_jobs_create_from_s3,
import_jobs_create_from_url,
Expand Down Expand Up @@ -814,28 +815,48 @@
},
"progress": {
"help": "Shows the progress of an import job.",
"extra_args": [
Argument(
"--cluster-id", type=str, required=True,
help="The cluster the import jobs belong to."
),
Argument(
"--import-job-id", type=str,
required=True,
help="The ID of the Import Job."
),
Argument(
"--limit", type=str, required=False,
help="The number of files returned."
"Use keywork 'ALL' to have no limit applied."
),
Argument(
"--offset", type=int, required=False,
help="The offset to skip before beginning to "
"return the files."
),
],
"resolver": import_job_progress,
"commands" : {
"summary": {
"help": "Global progress for an import job.",
"extra_args": [
Argument(
"--cluster-id", type=str, required=True,
help="The cluster the import jobs belong to."
),
Argument(
"--import-job-id", type=str,
required=True,
help="The ID of the Import Job."
),
],
"resolver": import_job_progress_summary,
},
"files": {
"help": "Progress file by file for an import job.",
"extra_args": [
Argument(
"--cluster-id", type=str, required=True,
help="The cluster the import jobs belong to."
),
Argument(
"--import-job-id", type=str,
required=True,
help="The ID of the Import Job."
),
Argument(
"--limit", type=str, required=False,
help="The number of files returned.Use "
"keywork 'ALL' to have no limit applied."
),
Argument(
"--offset", type=int, required=False,
help="The offset to skip before beginning to "
"return the files."
),
],
"resolver": import_job_progress_files,
},
},
},
},
},
Expand Down
35 changes: 24 additions & 11 deletions croud/clusters/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,28 @@ def import_jobs_list(args: Namespace) -> None:
)


def import_job_progress(args: Namespace) -> None:
def import_job_progress_summary(args: Namespace) -> None:
client = Client.from_args(args)

url = (
f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/"
)
data, errors = client.get(url)
print_response(
data=data.get("progress", {}) if data else {},
errors=errors,
keys=[
"percent",
"records",
"failed_records",
"total_records",
"total_files",
],
output_fmt=get_output_format(args),
)


def import_job_progress_files(args: Namespace) -> None:
client = Client.from_args(args)

params = {}
Expand All @@ -327,27 +348,19 @@ def import_job_progress(args: Namespace) -> None:
)
data, errors = client.get(url, params=params)
print_response(
data=data.get("progress", {}) if data else {},
data=data.get("progress", {}).get("files", []) if data else {},
errors=errors,
keys=[
"name",
"percent",
"records",
"failed_records",
"total_records",
"total_files",
"files",
],
output_fmt=get_output_format(args),
transforms={
"files": _format_files,
},
)


def _format_files(field):
return ",\n".join(str(f) for f in field) if field else None


def clusters_upgrade(args: Namespace) -> None:
body = {"crate_version": args.version}
client = Client.from_args(args)
Expand Down
51 changes: 44 additions & 7 deletions docs/commands/clusters.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -734,21 +734,58 @@ Example
:func: get_parser
:prog: croud
:path: clusters import-jobs progress
:nosubcommands:


``clusters import-jobs progress summary``
=========================================

.. argparse::
:module: croud.__main__
:func: get_parser
:prog: croud
:path: clusters import-jobs progress summary

Example
-------

.. code-block:: console
sh$ ❯ croud clusters import-jobs progress \
sh$ ❯ croud clusters import-jobs progress summary \
--cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \
--import-job-id 00de6048-3af6-41da-bfaa-661199d1c106
+-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+
| percent | records | failed_records | total_records | total_files | files |
|-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------|
| 100 | 891 | 0 | 891 | 2 | {'failed_records': 0, 'name': 'file1.csv', 'records': 800, 'total_records': 891}, |
| | | | | | {'failed_records': 0, 'name': 'file2.csv', 'records': 91, 'total_records': 891} |
+-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+
+-----------+-----------+------------------+-----------------+---------------+
| percent | records | failed_records | total_records | total_files |
|-----------+-----------+------------------+-----------------+---------------+
| 100 | 891 | 0 | 891 | 2 |
+-----------+-----------+------------------+-----------------+---------------+
``clusters import-jobs progress files``
=======================================

.. argparse::
:module: croud.__main__
:func: get_parser
:prog: croud
:path: clusters import-jobs progress files

Example
-------

.. code-block:: console
sh$ ❯ croud clusters import-jobs progress files \
--cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \
--import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \
--limit ALL
--offset 0
+-----------+-----------+-----------+------------------+-----------------+
| name | percent | records | failed_records | total_records |
|-----------+-----------+-----------+------------------+-----------------|
| file1.csv | 100 | 800 | 0 | 800 |
| file2.csv | 100 | 91 | 0 | 91 |
+-----------+-----------+-----------+------------------+-----------------+
``clusters export-jobs``
Expand Down
29 changes: 28 additions & 1 deletion tests/commands/test_clusters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1741,11 +1741,37 @@ def test_import_job_list(mock_request, output_format):
)


@mock.patch.object(Client, "request", return_value=({}, None))
def test_import_job_progress_summary(mock_request):
cluster_id = gen_uuid()
import_job_id = gen_uuid()

args = [
"croud",
"clusters",
"import-jobs",
"progress",
"summary",
"--cluster-id",
cluster_id,
"--import-job-id",
import_job_id,
]

call_command(*args)
assert_rest(
mock_request,
RequestMethod.GET,
f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/",
params=None,
)


@mock.patch.object(Client, "request", return_value=({}, None))
@pytest.mark.parametrize(
"params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}]
)
def test_import_job_progress(mock_request, params):
def test_import_job_progress_files(mock_request, params):
cluster_id = gen_uuid()
import_job_id = gen_uuid()

Expand All @@ -1754,6 +1780,7 @@ def test_import_job_progress(mock_request, params):
"clusters",
"import-jobs",
"progress",
"files",
"--cluster-id",
cluster_id,
"--import-job-id",
Expand Down

0 comments on commit 764b760

Please sign in to comment.