Merge pull request #161 from dblock/pagination

Added support for cursor pagination, closes #157.

CHANGELOG.md

@@ -1,5 +1,6 @@
 ### 0.9.2 (Next)
 
+* [#161](https://github.com/slack-ruby/slack-ruby-client/pull/161): Added support for cursor pagination - [@dblock](https://github.com/dblock).
 * Your contribution here.
 
 ### 0.9.1 (8/24/2017)

README.md

@@ -164,24 +164,37 @@ client = Slack::Web::Client.new(user_agent: 'Slack Ruby Client/1.0')
 
 The following settings are supported.
 
-setting      | description
--------------|-------------------------------------------------------------------------------------------------
-token        | Slack API token.
-user_agent   | User-agent, defaults to _Slack Ruby Client/version_.
-proxy        | Optional HTTP proxy.
-ca_path      | Optional SSL certificates path.
-ca_file      | Optional SSL certificates file.
-endpoint     | Slack endpoint, default is _https://slack.com/api_.
-logger       | Optional `Logger` instance that logs HTTP requests.
-timeout      | Optional open/read timeout in seconds.
-open_timeout | Optional connection open timeout in seconds.
+setting           | description
+------------------|-------------------------------------------------------------------------------------------------
+token             | Slack API token.
+user_agent        | User-agent, defaults to _Slack Ruby Client/version_.
+proxy             | Optional HTTP proxy.
+ca_path           | Optional SSL certificates path.
+ca_file           | Optional SSL certificates file.
+endpoint          | Slack endpoint, default is _https://slack.com/api_.
+logger            | Optional `Logger` instance that logs HTTP requests.
+timeout           | Optional open/read timeout in seconds.
+open_timeout      | Optional connection open timeout in seconds.
+default_page_size | Optional page size for paginated requests, default is _100_.
 
 You can also pass request options, including `timeout` and `open_timeout` into individual calls.
 
 ```ruby
 client.channels_list(request: { timeout: 180 })
 ```
 
+#### Pagination Support
+
+The Web client natively supports [cursor pagination](https://api.slack.com/docs/pagination#cursors) for methods that allow it, such as `users_list`. Supply a block and the client will make repeated requests adjusting the value of `cursor` with every response. The default limit is set to 100 and can be adjusted via `Slack::Web::Client.config.default_page_size` or by passing it directly into the API call.
+
+```ruby
+all_members = []
+client.users_list(presence: true, limit: 10) do |response|
+  all_members.concat(response.members)
+end
+all_members # many thousands of team members retrieved 10 at a time
+```
+
 ### RealTime Client
 
 The Real Time Messaging API is a WebSocket-based API that allows you to receive events from Slack in real time and send messages as user.

bin/commands/channels.rb

@@ -85,8 +85,10 @@
   g.desc 'Lists all channels in a Slack team.'
   g.long_desc %( Lists all channels in a Slack team. )
   g.command 'list' do |c|
+    c.flag 'cursor', desc: "Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first 'page' of the collection. See pagination for more detail."
     c.flag 'exclude_archived', desc: 'Exclude archived channels from the list.'
     c.flag 'exclude_members', desc: 'Exclude the members collection from each channel.'
+    c.flag 'limit', desc: "The maximum number of items to return. Fewer than the requested number of items may be returned, even if the end of the users list hasn't been reached."
     c.action do |_global_options, options, _args|
       puts JSON.dump($client.channels_list(options))
     end

bin/commands/im.rb

@@ -27,6 +27,8 @@
   g.desc 'Lists direct message channels for the calling user.'
   g.long_desc %( Lists direct message channels for the calling user. )
   g.command 'list' do |c|
+    c.flag 'cursor', desc: "Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first 'page' of the collection. See pagination for more detail."
+    c.flag 'limit', desc: "The maximum number of items to return. Fewer than the requested number of items may be returned, even if the end of the users list hasn't been reached."
     c.action do |_global_options, options, _args|
       puts JSON.dump($client.im_list(options))
     end

bin/commands/users.rb

@@ -39,7 +39,7 @@
   g.desc 'Lists all users in a Slack team.'
   g.long_desc %( Lists all users in a Slack team. )
   g.command 'list' do |c|
-    c.flag 'cursor', desc: "Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first \"page\" of the collection. See pagination for more detail."
+    c.flag 'cursor', desc: "Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first 'page' of the collection. See pagination for more detail."
     c.flag 'limit', desc: "The maximum number of items to return. Fewer than the requested number of items may be returned, even if the end of the users list hasn't been reached."
     c.flag 'presence', desc: 'Whether to include presence data in the output. Setting this to false improves performance, especially with large teams.'
     c.action do |_global_options, options, _args|

lib/slack-ruby-client.rb

@@ -26,6 +26,7 @@
 require_relative 'slack/web/faraday/request'
 require_relative 'slack/web/api/mixins'
 require_relative 'slack/web/api/endpoints'
+require_relative 'slack/web/pagination/cursor'
 require_relative 'slack/web/client'
 
 # RealTime API

lib/slack/web/api/endpoints/channels.rb

@@ -130,14 +130,24 @@ def channels_leave(options = {})
           #
           # Lists all channels in a Slack team.
           #
+          # @option options [Object] :cursor
+          #   Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first "page" of the collection. See pagination for more detail.
           # @option options [Object] :exclude_archived
           #   Exclude archived channels from the list.
           # @option options [Object] :exclude_members
           #   Exclude the members collection from each channel.
+          # @option options [Object] :limit
+          #   The maximum number of items to return. Fewer than the requested number of items may be returned, even if the end of the users list hasn't been reached.
           # @see https://api.slack.com/methods/channels.list
           # @see https://github.com/dblock/slack-api-ref/blob/master/methods/channels/channels.list.json
           def channels_list(options = {})
-            post('channels.list', options)
+            if block_given?
+              Pagination::Cursor.new(self, :channels_list, options).each do |page|
+                yield page
+              end
+            else
+              post('channels.list', options)
+            end
           end
 
           #

lib/slack/web/api/endpoints/im.rb

@@ -42,10 +42,20 @@ def im_history(options = {})
           #
           # Lists direct message channels for the calling user.
           #
+          # @option options [Object] :cursor
+          #   Paginate through collections of data by setting the cursor parameter to a next_cursor attribute returned by a previous request's response_metadata. Default value fetches the first "page" of the collection. See pagination for more detail.
+          # @option options [Object] :limit
+          #   The maximum number of items to return. Fewer than the requested number of items may be returned, even if the end of the users list hasn't been reached.
           # @see https://api.slack.com/methods/im.list
           # @see https://github.com/dblock/slack-api-ref/blob/master/methods/im/im.list.json
           def im_list(options = {})
-            post('im.list', options)
+            if block_given?
+              Pagination::Cursor.new(self, :im_list, options).each do |page|
+                yield page
+              end
+            else
+              post('im.list', options)
+            end
           end
 
           #

lib/slack/web/api/endpoints/users.rb

@@ -61,7 +61,13 @@ def users_info(options = {})
           # @see https://api.slack.com/methods/users.list
           # @see https://github.com/dblock/slack-api-ref/blob/master/methods/users/users.list.json
           def users_list(options = {})
-            post('users.list', options)
+            if block_given?
+              Pagination::Cursor.new(self, :users_list, options).each do |page|
+                yield page
+              end
+            else
+              post('users.list', options)
+            end
           end
 
           #

lib/slack/web/api/templates/command.erb

@@ -20,8 +20,8 @@ command '<%= group['name'].gsub(".", "_") %>' do |g|
   g.long_desc %( <%= data["desc"].split("\n").join(" ") %> )
   g.command '<%= name %>' do |c|
   <% data["args"].each do |arg_name, arg_v| %>
-    <% if arg_v["desc"].include?("'")  %>
-    c.flag '<%= arg_name %>', desc: "<%= arg_v["desc"] %>"
+    <% if arg_v["desc"].include?("'") %>
+    c.flag '<%= arg_name %>', desc: "<%= arg_v["desc"].gsub('"', '\'') %>"
     <% else %>
     c.flag '<%= arg_name %>', desc: '<%= arg_v["desc"] %>'
     <% end %>

lib/slack/web/api/templates/method.erb

@@ -39,7 +39,17 @@ module Slack
 <% if data['args']['user'] %>
             options = options.merge(user: users_id(options)['user']['id']) if options[:user]
 <% end %>
+<% if data['args'].keys.include?('cursor') %>
+            if block_given?
+              Pagination::Cursor.new(self, :<%= group.gsub(".", "_") %>_<%= name %>, options).each do |page|
+                yield page
+              end
+            else
+              post('<%= group %>.<%= name %>', options)
+            end
+<% else %>
             post('<%= group %>.<%= name %>', options)
+<% end %>
           end
 <% end %>
         end

lib/slack/web/config.rb

@@ -12,7 +12,8 @@ module Config
         :endpoint,
         :token,
         :timeout,
-        :open_timeout
+        :open_timeout,
+        :default_page_size
       ].freeze
 
       attr_accessor(*Config::ATTRIBUTES)
@@ -27,6 +28,7 @@ def reset
         self.logger = nil
         self.timeout = nil
         self.open_timeout = nil
+        self.default_page_size = 100
       end
     end
 

lib/slack/web/pagination/cursor.rb

@@ -0,0 +1,33 @@
+module Slack
+  module Web
+    module Api
+      module Pagination
+        class Cursor
+          include Enumerable
+
+          attr_reader :client
+          attr_reader :verb
+          attr_reader :params
+
+          def initialize(client, verb, params = {})
+            @client = client
+            @verb = verb
+            @params = params
+          end
+
+          def each
+            next_cursor = nil
+            loop do
+              query = { limit: client.default_page_size }.merge(params).merge(cursor: next_cursor)
+              response = client.send(verb, query)
+              yield response
+              break unless response.response_metadata
+              next_cursor = response.response_metadata.next_cursor
+              break if next_cursor.blank?
+            end
+          end
+        end
+      end
+    end
+  end
+end