diff --git a/bin/qds.py b/bin/qds.py index 77803380..24a1b13b 100755 --- a/bin/qds.py +++ b/bin/qds.py @@ -388,10 +388,10 @@ def cluster_list_action(clusterclass, args): result = clusterclass.show(arguments['cluster_id']) elif arguments['label'] is not None: result = clusterclass.show(arguments['label']) - elif arguments['state'] is not None: - result = clusterclass.list(state=arguments['state']) else: - result = clusterclass.list() + result = clusterclass.list(state=arguments['state'], + page=arguments['page'], + per_page=arguments['per_page']) print(json.dumps(result, indent=4)) return 0 diff --git a/qds_sdk/cluster.py b/qds_sdk/cluster.py index 27f5249c..504d9976 100755 --- a/qds_sdk/cluster.py +++ b/qds_sdk/cluster.py @@ -52,12 +52,17 @@ def _parse_list(cls, args): group.add_argument("--state", dest="state", action="store", choices=['up', 'down', 'pending', 'terminating'], help="list only clusters in the given state") + pagination_group = group.add_argument_group() + pagination_group.add_argument("--page", dest="page", action="store", type=int, + help="page number") + pagination_group.add_argument("--per-page", dest="per_page", action="store", type=int, + help="number of clusters to be retrieved per page") arguments = argparser.parse_args(args) return vars(arguments) @classmethod - def list(cls, state=None): + def list(cls, state=None, page=None, per_page=None): """ List existing clusters present in your account. @@ -68,14 +73,28 @@ def list(cls, state=None): List of clusters satisfying the given criteria """ conn = Qubole.agent() + params = {} + if page: + params['page'] = page + if per_page: + params['per_page'] = per_page + if (params.get('page') or params.get('per_page')) and Qubole.version == 'v1.2': + log.warn("Pagination is not supported with API v1.2. Fetching all clusters.") + params = None if not params else params + cluster_list = conn.get(cls.rest_entity_path, params=params) if state is None: - return conn.get(cls.rest_entity_path) + return cluster_list elif state is not None: - cluster_list = conn.get(cls.rest_entity_path) result = [] - for cluster in cluster_list: - if state.lower() == cluster['cluster']['state'].lower(): - result.append(cluster) + if Qubole.version == 'v1.2': + for cluster in cluster_list: + if state.lower() == cluster['cluster']['state'].lower(): + result.append(cluster) + elif Qubole.version == 'v1.3': + cluster_list = cluster_list['clusters'] + for cluster in cluster_list: + if state.lower() == cluster['state'].lower(): + result.append(cluster) return result @classmethod diff --git a/qds_sdk/clusterv2.py b/qds_sdk/clusterv2.py index 11aa9c3d..95e7905a 100755 --- a/qds_sdk/clusterv2.py +++ b/qds_sdk/clusterv2.py @@ -64,7 +64,8 @@ def run(args): if args[0] in ["create", "clone", "update"]: ClusterCmdLine.get_cluster_create_clone_update(arguments, args[0]) else: - return arguments.func(arguments.label, arguments.cluster_id, arguments.state) + return arguments.func(arguments.label, arguments.cluster_id, arguments.state, + arguments.page, arguments.per_page) @staticmethod def get_cluster_create_clone_update(arguments, action): @@ -401,6 +402,12 @@ def list_info_parser(argparser, action): argparser.add_argument("--state", dest="state", choices=['invalid', 'up', 'down', 'pending', 'terminating'], help="State of the cluster") + argparser.add_argument("--page", dest="page", + type=int, + help="Page number") + argparser.add_argument("--per-page", dest="per_page", + type=int, + help="Number of clusters to be retrieved per page") @staticmethod def cluster_info_parser(argparser, action): @@ -665,12 +672,14 @@ def clone(cls, cluster_id_label, cluster_info): return conn.post(cls.element_path(cluster_id_label) + '/clone', data=cluster_info) @classmethod - def list(cls, label=None, cluster_id=None, state=None): + def list(cls, label=None, cluster_id=None, state=None, page=None, per_page=None): """ List existing clusters present in your account. Kwargs: `state`: list only those clusters which are in this state + `page`: page number + `per_page`: number of clusters to be retrieved per page Returns: List of clusters satisfying the given criteria @@ -679,11 +688,17 @@ def list(cls, label=None, cluster_id=None, state=None): return cls.show(cluster_id) if label is not None: return cls.show(label) + params = {} + if page: + params['page'] = page + if per_page: + params['per_page'] = per_page + params = None if not params else params conn = Qubole.agent(version="v2") cluster_list = conn.get(cls.rest_entity_path) if state is None: # return the complete list since state is None - return conn.get(cls.rest_entity_path) + return conn.get(cls.rest_entity_path, params=params) # filter clusters based on state result = [] if 'clusters' in cluster_list: diff --git a/tests/test_cluster.py b/tests/test_cluster.py index d4055a99..d16764ea 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -73,6 +73,36 @@ def test_state_invalid(self): with self.assertRaises(SystemExit): qds.main() + def test_page(self): + sys.argv = ['qds.py', 'cluster', 'list', '--page', '2'] + print_command() + Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}]) + params = {'page': 2} + qds.main() + Connection._api_call.assert_called_with("GET", "clusters", params=params) + + def test_page_invalid(self): + sys.argv = ['qds.py', 'cluster', 'list', '--page', 'string_value'] + print_command() + Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}]) + with self.assertRaises(SystemExit): + qds.main() + + def test_per_page(self): + sys.argv = ['qds.py', 'cluster', 'list', '--per-page', '5'] + print_command() + Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}]) + params = {'per_page': 5} + qds.main() + Connection._api_call.assert_called_with("GET", "clusters", params=params) + + def test_per_page_invalid(self): + sys.argv = ['qds.py', 'cluster', 'list', '--per-page', 'string_value'] + print_command() + Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}]) + with self.assertRaises(SystemExit): + qds.main() + def test_connection(self): sys.argv = ['qds.py', 'cluster', 'list'] print_command() diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index aabf6694..e8b4de78 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -841,6 +841,37 @@ def test_state_invalid(self): qds.main() Connection._api_call.assert_called_with('GET', 'clusters', params=None) + def test_page(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--page', '2'] + Qubole.cloud = None + params = {"page": 2} + print_command() + Connection._api_call = Mock(return_value=[{"cluster" : {"state" : "up"}}]) + qds.main() + Connection._api_call.assert_called_with('GET', 'clusters', params=params) + + def test_page_invalid(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--page', 'string_value'] + print_command() + with self.assertRaises(SystemExit): + qds.main() + + def test_per_page(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--per-page', '5'] + Qubole.cloud = None + params = {"per_page": 5} + print_command() + Connection._api_call = Mock(return_value=[{"cluster" : {"state" : "up"}}]) + qds.main() + Connection._api_call.assert_called_with('GET', 'clusters', params=params) + + def test_per_page_invalid(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--per-page', 'string_value'] + print_command() + with self.assertRaises(SystemExit): + qds.main() + + class TestClusterShow(QdsCliTestCase): def test_connection(self):