From 764b7604ee11eead618c3b82ff20a835e7e91aba Mon Sep 17 00:00:00 2001 From: Pauline Laharanne Date: Tue, 17 Oct 2023 10:43:14 +0200 Subject: [PATCH] split progress in two commands --- croud/__main__.py | 67 ++++++++++++++++++++++----------- croud/clusters/commands.py | 35 +++++++++++------ docs/commands/clusters.rst | 51 +++++++++++++++++++++---- tests/commands/test_clusters.py | 29 +++++++++++++- 4 files changed, 140 insertions(+), 42 deletions(-) diff --git a/croud/__main__.py b/croud/__main__.py index 07dbc6ba..95a2e434 100644 --- a/croud/__main__.py +++ b/croud/__main__.py @@ -50,7 +50,8 @@ export_jobs_create, export_jobs_delete, export_jobs_list, - import_job_progress, + import_job_progress_files, + import_job_progress_summary, import_jobs_create_from_file, import_jobs_create_from_s3, import_jobs_create_from_url, @@ -814,28 +815,48 @@ }, "progress": { "help": "Shows the progress of an import job.", - "extra_args": [ - Argument( - "--cluster-id", type=str, required=True, - help="The cluster the import jobs belong to." - ), - Argument( - "--import-job-id", type=str, - required=True, - help="The ID of the Import Job." - ), - Argument( - "--limit", type=str, required=False, - help="The number of files returned." - "Use keywork 'ALL' to have no limit applied." - ), - Argument( - "--offset", type=int, required=False, - help="The offset to skip before beginning to " - "return the files." - ), - ], - "resolver": import_job_progress, + "commands" : { + "summary": { + "help": "Global progress for an import job.", + "extra_args": [ + Argument( + "--cluster-id", type=str, required=True, + help="The cluster the import jobs belong to." + ), + Argument( + "--import-job-id", type=str, + required=True, + help="The ID of the Import Job." + ), + ], + "resolver": import_job_progress_summary, + }, + "files": { + "help": "Progress file by file for an import job.", + "extra_args": [ + Argument( + "--cluster-id", type=str, required=True, + help="The cluster the import jobs belong to." + ), + Argument( + "--import-job-id", type=str, + required=True, + help="The ID of the Import Job." + ), + Argument( + "--limit", type=str, required=False, + help="The number of files returned.Use " + "keywork 'ALL' to have no limit applied." + ), + Argument( + "--offset", type=int, required=False, + help="The offset to skip before beginning to " + "return the files." + ), + ], + "resolver": import_job_progress_files, + }, + }, }, }, }, diff --git a/croud/clusters/commands.py b/croud/clusters/commands.py index affc5251..e6d427df 100644 --- a/croud/clusters/commands.py +++ b/croud/clusters/commands.py @@ -313,7 +313,28 @@ def import_jobs_list(args: Namespace) -> None: ) -def import_job_progress(args: Namespace) -> None: +def import_job_progress_summary(args: Namespace) -> None: + client = Client.from_args(args) + + url = ( + f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/" + ) + data, errors = client.get(url) + print_response( + data=data.get("progress", {}) if data else {}, + errors=errors, + keys=[ + "percent", + "records", + "failed_records", + "total_records", + "total_files", + ], + output_fmt=get_output_format(args), + ) + + +def import_job_progress_files(args: Namespace) -> None: client = Client.from_args(args) params = {} @@ -327,27 +348,19 @@ def import_job_progress(args: Namespace) -> None: ) data, errors = client.get(url, params=params) print_response( - data=data.get("progress", {}) if data else {}, + data=data.get("progress", {}).get("files", []) if data else {}, errors=errors, keys=[ + "name", "percent", "records", "failed_records", "total_records", - "total_files", - "files", ], output_fmt=get_output_format(args), - transforms={ - "files": _format_files, - }, ) -def _format_files(field): - return ",\n".join(str(f) for f in field) if field else None - - def clusters_upgrade(args: Namespace) -> None: body = {"crate_version": args.version} client = Client.from_args(args) diff --git a/docs/commands/clusters.rst b/docs/commands/clusters.rst index 9aeb3158..f81db816 100644 --- a/docs/commands/clusters.rst +++ b/docs/commands/clusters.rst @@ -734,21 +734,58 @@ Example :func: get_parser :prog: croud :path: clusters import-jobs progress + :nosubcommands: + + +``clusters import-jobs progress summary`` +========================================= + +.. argparse:: + :module: croud.__main__ + :func: get_parser + :prog: croud + :path: clusters import-jobs progress summary Example ------- .. code-block:: console - sh$ ❯ croud clusters import-jobs progress \ + sh$ ❯ croud clusters import-jobs progress summary \ --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 - +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ - | percent | records | failed_records | total_records | total_files | files | - |-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------| - | 100 | 891 | 0 | 891 | 2 | {'failed_records': 0, 'name': 'file1.csv', 'records': 800, 'total_records': 891}, | - | | | | | | {'failed_records': 0, 'name': 'file2.csv', 'records': 91, 'total_records': 891} | - +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ + +-----------+-----------+------------------+-----------------+---------------+ + | percent | records | failed_records | total_records | total_files | + |-----------+-----------+------------------+-----------------+---------------+ + | 100 | 891 | 0 | 891 | 2 | + +-----------+-----------+------------------+-----------------+---------------+ + + +``clusters import-jobs progress files`` +======================================= + +.. argparse:: + :module: croud.__main__ + :func: get_parser + :prog: croud + :path: clusters import-jobs progress files + +Example +------- + +.. code-block:: console + + sh$ ❯ croud clusters import-jobs progress files \ + --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \ + --limit ALL + --offset 0 + +-----------+-----------+-----------+------------------+-----------------+ + | name | percent | records | failed_records | total_records | + |-----------+-----------+-----------+------------------+-----------------| + | file1.csv | 100 | 800 | 0 | 800 | + | file2.csv | 100 | 91 | 0 | 91 | + +-----------+-----------+-----------+------------------+-----------------+ ``clusters export-jobs`` diff --git a/tests/commands/test_clusters.py b/tests/commands/test_clusters.py index c23d208d..b2f62a59 100644 --- a/tests/commands/test_clusters.py +++ b/tests/commands/test_clusters.py @@ -1741,11 +1741,37 @@ def test_import_job_list(mock_request, output_format): ) +@mock.patch.object(Client, "request", return_value=({}, None)) +def test_import_job_progress_summary(mock_request): + cluster_id = gen_uuid() + import_job_id = gen_uuid() + + args = [ + "croud", + "clusters", + "import-jobs", + "progress", + "summary", + "--cluster-id", + cluster_id, + "--import-job-id", + import_job_id, + ] + + call_command(*args) + assert_rest( + mock_request, + RequestMethod.GET, + f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/", + params=None, + ) + + @mock.patch.object(Client, "request", return_value=({}, None)) @pytest.mark.parametrize( "params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}] ) -def test_import_job_progress(mock_request, params): +def test_import_job_progress_files(mock_request, params): cluster_id = gen_uuid() import_job_id = gen_uuid() @@ -1754,6 +1780,7 @@ def test_import_job_progress(mock_request, params): "clusters", "import-jobs", "progress", + "files", "--cluster-id", cluster_id, "--import-job-id",