SDK-330: Add page and per-page option to Cluster and ClusterV2 (#257)

bin/qds.py

@@ -388,10 +388,10 @@ def cluster_list_action(clusterclass, args):
         result = clusterclass.show(arguments['cluster_id'])
     elif arguments['label'] is not None:
         result = clusterclass.show(arguments['label'])
-    elif arguments['state'] is not None:
-        result = clusterclass.list(state=arguments['state'])
     else:
-        result = clusterclass.list()
+        result = clusterclass.list(state=arguments['state'],
+                                   page=arguments['page'],
+                                   per_page=arguments['per_page'])
     print(json.dumps(result, indent=4))
     return 0
 

qds_sdk/cluster.py

@@ -52,12 +52,17 @@ def _parse_list(cls, args):
         group.add_argument("--state", dest="state", action="store",
                            choices=['up', 'down', 'pending', 'terminating'],
                            help="list only clusters in the given state")
+        pagination_group = group.add_argument_group()
+        pagination_group.add_argument("--page", dest="page", action="store", type=int,
+                           help="page number")
+        pagination_group.add_argument("--per-page", dest="per_page", action="store", type=int,
+                           help="number of clusters to be retrieved per page")
 
         arguments = argparser.parse_args(args)
         return vars(arguments)
 
     @classmethod
-    def list(cls, state=None):
+    def list(cls, state=None, page=None, per_page=None):
         """
         List existing clusters present in your account.
 
@@ -68,14 +73,28 @@ def list(cls, state=None):
             List of clusters satisfying the given criteria
         """
         conn = Qubole.agent()
+        params = {}
+        if page:
+            params['page'] = page
+        if per_page:
+            params['per_page'] = per_page
+        if (params.get('page') or params.get('per_page')) and Qubole.version == 'v1.2':
+            log.warn("Pagination is not supported with API v1.2. Fetching all clusters.")
+        params = None if not params else params
+        cluster_list = conn.get(cls.rest_entity_path, params=params)
         if state is None:
-            return conn.get(cls.rest_entity_path)
+            return cluster_list
         elif state is not None:
-            cluster_list = conn.get(cls.rest_entity_path)
             result = []
-            for cluster in cluster_list:
-                if state.lower() == cluster['cluster']['state'].lower():
-                    result.append(cluster)
+            if Qubole.version == 'v1.2':
+                for cluster in cluster_list:
+                    if state.lower() == cluster['cluster']['state'].lower():
+                        result.append(cluster)
+            elif Qubole.version == 'v1.3':
+                cluster_list = cluster_list['clusters']
+                for cluster in cluster_list:
+                    if state.lower() == cluster['state'].lower():
+                        result.append(cluster)
             return result
 
     @classmethod

qds_sdk/clusterv2.py

@@ -64,7 +64,8 @@ def run(args):
         if args[0] in ["create", "clone", "update"]:
             ClusterCmdLine.get_cluster_create_clone_update(arguments, args[0])
         else:
-            return arguments.func(arguments.label, arguments.cluster_id, arguments.state)
+            return arguments.func(arguments.label, arguments.cluster_id, arguments.state,
+                                  arguments.page, arguments.per_page)
 
     @staticmethod
     def get_cluster_create_clone_update(arguments, action):
@@ -401,6 +402,12 @@ def list_info_parser(argparser, action):
         argparser.add_argument("--state", dest="state",
                                choices=['invalid', 'up', 'down', 'pending', 'terminating'],
                                help="State of the cluster")
+        argparser.add_argument("--page", dest="page",
+                               type=int,
+                               help="Page number")
+        argparser.add_argument("--per-page", dest="per_page",
+                               type=int,
+                               help="Number of clusters to be retrieved per page")
 
     @staticmethod
     def cluster_info_parser(argparser, action):
@@ -665,12 +672,14 @@ def clone(cls, cluster_id_label, cluster_info):
         return conn.post(cls.element_path(cluster_id_label) + '/clone', data=cluster_info)
 
     @classmethod
-    def list(cls, label=None, cluster_id=None, state=None):
+    def list(cls, label=None, cluster_id=None, state=None, page=None, per_page=None):
         """
         List existing clusters present in your account.
 
         Kwargs:
             `state`: list only those clusters which are in this state
+            `page`: page number
+            `per_page`: number of clusters to be retrieved per page
 
         Returns:
             List of clusters satisfying the given criteria
@@ -679,11 +688,17 @@ def list(cls, label=None, cluster_id=None, state=None):
             return cls.show(cluster_id)
         if label is not None:
             return cls.show(label)
+        params = {}
+        if page:
+            params['page'] = page
+        if per_page:
+            params['per_page'] = per_page
+        params = None if not params else params
         conn = Qubole.agent(version="v2")
         cluster_list = conn.get(cls.rest_entity_path)
         if state is None:
             # return the complete list since state is None
-            return conn.get(cls.rest_entity_path)
+            return conn.get(cls.rest_entity_path, params=params)
         # filter clusters based on state
         result = []
         if 'clusters' in cluster_list:

tests/test_cluster.py

@@ -73,6 +73,36 @@ def test_state_invalid(self):
         with self.assertRaises(SystemExit):
             qds.main()
 
+    def test_page(self):
+        sys.argv = ['qds.py', 'cluster', 'list', '--page', '2']
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}])
+        params = {'page': 2}
+        qds.main()
+        Connection._api_call.assert_called_with("GET", "clusters", params=params)
+
+    def test_page_invalid(self):
+        sys.argv = ['qds.py', 'cluster', 'list', '--page', 'string_value']
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}])
+        with self.assertRaises(SystemExit):
+            qds.main()
+
+    def test_per_page(self):
+        sys.argv = ['qds.py', 'cluster', 'list', '--per-page', '5']
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}])
+        params = {'per_page': 5}
+        qds.main()
+        Connection._api_call.assert_called_with("GET", "clusters", params=params)
+
+    def test_per_page_invalid(self):
+        sys.argv = ['qds.py', 'cluster', 'list', '--per-page', 'string_value']
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster": {"state": "up"}}])
+        with self.assertRaises(SystemExit):
+            qds.main()
+
     def test_connection(self):
         sys.argv = ['qds.py', 'cluster', 'list']
         print_command()

tests/test_clusterv2.py

@@ -841,6 +841,37 @@ def test_state_invalid(self):
         qds.main()
         Connection._api_call.assert_called_with('GET', 'clusters', params=None)
 
+    def test_page(self):
+        sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--page', '2']
+        Qubole.cloud = None
+        params = {"page": 2}
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster" : {"state" : "up"}}])
+        qds.main()
+        Connection._api_call.assert_called_with('GET', 'clusters', params=params)
+
+    def test_page_invalid(self):
+        sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--page', 'string_value']
+        print_command()
+        with self.assertRaises(SystemExit):
+            qds.main()
+
+    def test_per_page(self):
+        sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--per-page', '5']
+        Qubole.cloud = None
+        params = {"per_page": 5}
+        print_command()
+        Connection._api_call = Mock(return_value=[{"cluster" : {"state" : "up"}}])
+        qds.main()
+        Connection._api_call.assert_called_with('GET', 'clusters', params=params)
+
+    def test_per_page_invalid(self):
+        sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'list', '--per-page', 'string_value']
+        print_command()
+        with self.assertRaises(SystemExit):
+            qds.main()
+
+
 class TestClusterShow(QdsCliTestCase):
 
     def test_connection(self):