Sitemap & Google Scholar

Gemfile

@@ -63,6 +63,7 @@ group :development, :test do
   gem 'sdoc', require: false
 
   gem 'capybara', '~> 2.17'
+  gem 'nokogiri'
   gem 'selenium-webdriver', require: false
 
   gem 'pry'

Gemfile.lock

@@ -416,6 +416,7 @@ DEPENDENCIES
   listen (>= 3.0.5, < 3.2)
   minitest-hooks
   mysql2 (>= 0.3.18, < 0.5)
+  nokogiri
   omniauth
   omniauth-saml
   pry
@@ -446,4 +447,4 @@ DEPENDENCIES
   wicked
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

app/controllers/robots_controller.rb

@@ -0,0 +1,10 @@
+class RobotsController < ApplicationController
+
+  skip_after_action :verify_authorized
+
+  def robots
+    respond_to :text
+    expires_in 6.hours, public: true
+  end
+
+end

app/controllers/sitemap_controller.rb

@@ -0,0 +1,28 @@
+class SitemapController < ApplicationController
+
+  skip_after_action :verify_authorized
+
+  def index; end
+
+  def communities
+    @communities = Community.all
+    # TODO: consider using Rollbar to catch this kind of thing
+    logger.warn 'communities sitemap should contain less than 50,000 targets' if @communities.count > 50_000
+  end
+
+  def collections
+    @collections = Collection.all
+    logger.warn 'collections sitemap should contain less than 50,000 targets' if @collections.count > 50_000
+  end
+
+  def items
+    @items = Item.public
+    logger.warn 'items sitemap should contain less than 50,000 targets' if @items.count > 50_000
+  end
+
+  def theses
+    @theses = Thesis.public
+    logger.warn 'thesis sitemap should contain less than 50,000 targets' if @theses.count > 50_000
+  end
+
+end

app/helpers/items_helper.rb

@@ -16,4 +16,9 @@ def license_link(license)
     text ||= license
     link_to(text, license)
   end
+
+  def authors(object)
+    return [object.dissertant] if object.respond_to? :dissertant
+    return object.creators if object.respond_to? :creators
+  end
 end

app/models/concerns/item_properties.rb

@@ -116,6 +116,20 @@ def add_files(files)
                                                   else
                                                     File.basename(file)
                                                   end
+            # Store file properties in the format required by the sitemap
+            # for quick and easy retrieval -- nobody wants to wait 36hrs for this!
+            unlocked_fileset.sitemap_link = "<rs:ln \
+href=\"#{Rails.application.routes.url_helpers.url_for(controller: :file_sets,
+                                                      action: :download,
+                                                      id: unlocked_fileset.id,
+                                                      file_name: unlocked_fileset.contained_filename,
+                                                      only_path: true)}\" \
+rel=\"content\" \
+hash=\"#{unlocked_fileset.original_file.checksum.algorithm.downcase}:"\
+"#{unlocked_fileset.original_file.checksum.value}\" \
+length=\"#{unlocked_fileset.original_file.size}\" \
+type=\"#{unlocked_fileset.original_file.mime_type}\"\
+/>"
             unlocked_fileset.save!
             self.members += [unlocked_fileset]
             # pull in hydra derivatives, set temp file base
@@ -134,6 +148,10 @@ def display_attribute_names
     def valid_visibilities
       super + [VISIBILITY_EMBARGO]
     end
+
+    def public
+      where(visibility: JupiterCore::VISIBILITY_PUBLIC)
+    end
   end
 
   def doi_url

app/models/file_set.rb

@@ -6,6 +6,7 @@ class FileSet < JupiterCore::LockedLdpObject
   ldp_object_includes Hydra::Works::FileSetBehavior
 
   has_attribute :contained_filename, ::RDF::Vocab::DC.title, solrize_for: :exact_match
+  has_attribute :sitemap_link, ::TERMS[:ual].sitemap_link, solrize_for: :exact_match
 
   belongs_to :item, using_existing_association: :member_of_collections
 

app/views/items/_google_scholar_metadata.html.erb

@@ -0,0 +1,13 @@
+<!-- Google Scholar metadata -->
+<meta name="citation_title" content="<%= @item.title%>"/>
+<% authors(@item).each do |author| %>
+<meta name="citation_author" content="<%= author %>"/>
+<% end %>
+<% if @item.sort_year.present? %>
+<meta name="citation_publication_date" content="<%= @item.sort_year %>"/>
+<% end %>
+<% if @item.file_sets.count > 0 %>
+<meta name="citation_pdf_url" content="<%= fileset_uri(@item.file_sets.first, :download) %>"/>
+<% end %>
+<meta name="dc.identifier" content="<%= @item.doi %>"/>
+<meta name="citation_doi" content="<%= @item.doi %>"/>

app/views/items/show.html.erb

@@ -1,3 +1,7 @@
+<% content_for :head do %>
+  <%= render partial: 'google_scholar_metadata' %>
+<% end %>
+
 <% page_title(@item.title) %>
 
 <%# Breadcrumbs here are for 'Home->Search->Item' %>

app/views/robots/robots.text.erb

@@ -0,0 +1,7 @@
+# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
+Sitemap: https://era.library.ualberta.ca/sitemap.xml
+
+<% unless Rails.configuration.allow_crawlers %>
+User-agent: *
+Disallow: /
+<% end %>

app/views/sitemap/_object.xml.builder

@@ -0,0 +1,27 @@
+xml.instruct! :xml, version: '1.0'
+cache 'sitemap', expires_in: 24.hours do
+  xml.urlset(xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
+             'xmlns:rs' => 'http://www.openarchives.org/rs/terms/') do
+    xml.rs :md, capability: 'resourcelist', at: Time.current.iso8601
+    xml.url do
+      xml.loc        root_url
+      xml.lastmod    Time.current.iso8601
+      xml.changefreq 'weekly'
+      xml.priority   1
+      xml.rs :md, type: 'text/html'
+    end
+
+    objects.each do |object|
+      xml.url do
+        xml.loc item_url(object)
+        xml.lastmod object.updated_at
+        xml.changefreq 'weekly'
+        xml.priority   1
+        xml.rs :md, type: 'text/html'
+        object.file_sets.each do |file_set|
+          xml << "\t#{file_set.sitemap_link}\n"
+        end
+      end
+    end
+  end
+end

app/views/sitemap/collections.xml.builder

@@ -0,0 +1,20 @@
+xml.instruct! :xml, version: '1.0'
+cache 'sitemap', expires_in: 24.hours do
+  xml.urlset(xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9') do
+    xml.url do
+      xml.loc        root_url
+      xml.lastmod    Time.current.utc.iso8601
+      xml.changefreq 'weekly'
+      xml.priority   1
+    end
+
+    @collections.each do |collection|
+      xml.url do
+        xml.loc community_collection_url(collection.community, collection)
+        xml.lastmod collection.updated_at
+        xml.changefreq 'weekly'
+        xml.priority   1
+      end
+    end
+  end
+end

app/views/sitemap/communities.xml.builder

@@ -0,0 +1,20 @@
+xml.instruct! :xml, version: '1.0'
+cache 'sitemap', expires_in: 24.hours do
+  xml.urlset(xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9') do
+    xml.url do
+      xml.loc        root_url
+      xml.lastmod    Time.current.utc.iso8601
+      xml.changefreq 'weekly'
+      xml.priority   1
+    end
+
+    @communities.each do |community|
+      xml.url do
+        xml.loc community_url(community)
+        xml.lastmod community.updated_at
+        xml.changefreq 'weekly'
+        xml.priority   1
+      end
+    end
+  end
+end

app/views/sitemap/index.xml.builder

@@ -0,0 +1,8 @@
+xml.instruct! :xml, version: '1.0'
+xml.sitemapindex xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9' do
+  [:items, :theses, :collections, :communities].each do |category|
+    xml.sitemap do
+      xml.loc url_for(action: category, controller: 'sitemap', only_path: false)
+    end
+  end
+end

app/views/sitemap/items.xml.builder

@@ -0,0 +1 @@
+xml << render(partial: 'object', locals: { objects: @items })

app/views/sitemap/theses.xml.builder

@@ -0,0 +1 @@
+xml << render(partial: 'object', locals: { objects: @theses })

config/application.rb

@@ -33,5 +33,8 @@ class Application < Rails::Application
     # Run skylight in UAT for performance metric monitoring pre-launch
     config.skylight.environments += ['uat']
 
+    # we can tell crawlers to go away during the beta launch phase
+    config.allow_crawlers = ENV['RAILS_ALLOW_CRAWLERS'].present?
+
   end
 end

config/routes.rb

@@ -62,5 +62,14 @@
     mount Sidekiq::Web => '/sidekiq', constraints: AdminConstraint.new
   end
 
+  get 'sitemap.xml', to: 'sitemap#index', defaults: { format: :xml }, as: :sitemapindex
+  get 'sitemap-communities.xml', to: 'sitemap#communities', defaults: { format: :xml }, as: :communities_sitemap
+  get 'sitemap-collections.xml', to: 'sitemap#collections', defaults: { format: :xml }, as: :collections_sitemap
+  get 'sitemap-items.xml', to: 'sitemap#items', defaults: { format: :xml }, as: :items_sitemap
+  get 'sitemap-theses.xml', to: 'sitemap#theses', defaults: { format: :xml }
+
+  # Dynamic robots.txt
+  get 'robots.txt' => 'robots#robots'
+
   root to: 'welcome#index'
 end