Support passing sitemap: :robots and string paths

lib/spidr/agent.rb

@@ -223,9 +223,7 @@ def initialize(options={})
         initialize_robots
       end
 
-      if options.fetch(:sitemap,false)
-        initialize_sitemap
-      end
+      initialize_sitemap(options)
 
       yield self if block_given?
     end

lib/spidr/agent/sitemap.rb

@@ -16,8 +16,8 @@ class Agent
     #
     # Initializes the sitemap fetcher.
     #
-    def initialize_sitemap
-      @sitemap = true
+    def initialize_sitemap(options)
+      @sitemap = options.fetch(:sitemap, false)
     end
 
     #
@@ -31,12 +31,21 @@ def sitemap_urls(url)
       return [] unless @sitemap
       base_url = to_base_url(url)
 
-      if @robots
-        if urls = @robots.other_values(base_url)['Sitemap']
+      # Support passing sitemap: '/path/to/sitemap.xml'
+      if @sitemap.is_a?(String)
+        sitemap_path = @sitemap
+        sitemap_path = "/#{@sitemap}" unless @sitemap.start_with?('/')
+        return get_sitemap_urls(url: "#{base_url}#{sitemap_path}")
+      end
+
+      # Check /robots.txt
+      if sitemap_robots
+        if urls = sitemap_robots.other_values(base_url)['Sitemap']
           return urls.flat_map { |u| get_sitemap_urls(url: u) }
         end
       end
 
+      # Check for sitemap.xml in common locations
       COMMON_SITEMAP_LOCATIONS.each do |path|
         if (page = get_page("#{base_url}/#{path}")).code == 200
           return get_sitemap_urls(page: page)
@@ -48,6 +57,17 @@ def sitemap_urls(url)
 
     private
 
+    def sitemap_robots
+      return @robots if @robots
+      return unless @sitemap == :robots
+
+      unless Object.const_defined?(:Robots)
+        raise(ArgumentError,":robots option given but unable to require 'robots' gem")
+      end
+
+      Robots.new(@user_agent)
+    end
+
     def get_sitemap_urls(url: nil, page: nil)
       page = get_page(url) if page.nil?
       return [] unless page

spec/agent/sitemap_spec.rb

@@ -5,6 +5,45 @@
 
 describe Agent do
   describe "sitemap" do
+    context "parh from sitemap option" do
+      include_context "example App"
+
+      subject { described_class.new(host: host, sitemap: 'my-sitemap.xml') }
+
+      app do
+        before do
+          content_type 'application/xml'
+        end
+
+        get '/my-sitemap.xml' do
+          <<-SITEMAP_XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url>
+               <loc>http://example.com/</loc>
+            </url>
+             <url>
+                <loc>http://example.com/some-path</loc>
+             </url>
+          </urlset>
+          SITEMAP_XML
+        end
+      end
+
+      before do
+        stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+      end
+
+      it 'should fetch all URLs in sitemap' do
+        urls = subject.sitemap_urls('http://example.com')
+        expected = [
+          URI('http://example.com/'),
+          URI('http://example.com/some-path')
+        ]
+        expect(urls).to eq(expected)
+      end
+    end
+
     context "from common sitemap index path" do
       include_context "example App"