Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Nest generators under the WebmentionIO module #108

Merged
merged 1 commit into from
Sep 15, 2018
Merged

Nest generators under the WebmentionIO module #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
238 changes: 120 additions & 118 deletions lib/jekyll/generators/gather_webmentions.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,155 +10,157 @@
require "time"

module Jekyll
class GatherWebmentions < Generator
safe true
priority :high

def generate(site)
@site = site
@site_url = site.config["url"].to_s

if @site_url.include? "localhost"
Jekyll::WebmentionIO.log "msg", "Webmentions won’t be gathered on localhost."
return
end
module WebmentionIO
class GatherWebmentions < Generator
safe true
priority :high

if @site.config.dig("webmentions", "pause_lookups") == true
WebmentionIO.log "msg", "Webmention gathering is currently paused."
return
end
def generate(site)
@site = site
@site_url = site.config["url"].to_s

WebmentionIO.log "msg", "Beginning to gather webmentions of your posts. This may take a while."
if @site_url.include? "localhost"
Jekyll::WebmentionIO.log "msg", "Webmentions won’t be gathered on localhost."
return
end

WebmentionIO.api_path = "mentions"
# add an arbitrarily high perPage to trump pagination
WebmentionIO.api_suffix = "&perPage=9999"
if @site.config.dig("webmentions", "pause_lookups") == true
WebmentionIO.log "msg", "Webmention gathering is currently paused."
return
end

@cached_webmentions = WebmentionIO.read_cached_webmentions "incoming"
WebmentionIO.log "msg", "Beginning to gather webmentions of your posts. This may take a while."

@lookups = WebmentionIO.read_lookup_dates
WebmentionIO.api_path = "mentions"
# add an arbitrarily high perPage to trump pagination
WebmentionIO.api_suffix = "&perPage=9999"

posts = WebmentionIO.gather_documents(@site)
posts.each do |post|
check_for_webmentions(post)
end
@cached_webmentions = WebmentionIO.read_cached_webmentions "incoming"

WebmentionIO.cache_lookup_dates @lookups
@lookups = WebmentionIO.read_lookup_dates

WebmentionIO.cache_webmentions "incoming", @cached_webmentions
end # generate
posts = WebmentionIO.gather_documents(@site)
posts.each do |post|
check_for_webmentions(post)
end

private
WebmentionIO.cache_lookup_dates @lookups

def check_for_webmentions(post)
WebmentionIO.log "info", "Checking for webmentions of #{post.url}."
WebmentionIO.cache_webmentions "incoming", @cached_webmentions
end # generate

last_webmention = @cached_webmentions.dig(post.url, @cached_webmentions.dig(post.url)&.keys&.last)
private

# get the last webmention
last_lookup = if @lookups[post.url]
@lookups[post.url]
elsif last_webmention
Date.parse last_webmention.dig("raw", "verified_date")
end
def check_for_webmentions(post)
WebmentionIO.log "info", "Checking for webmentions of #{post.url}."

# should we throttle?
if post.respond_to? "date" # Some docs have no date
if last_lookup && WebmentionIO.post_should_be_throttled?(post, post.date, last_lookup)
WebmentionIO.log "info", "Throttling this post."
return
last_webmention = @cached_webmentions.dig(post.url, @cached_webmentions.dig(post.url)&.keys&.last)

# get the last webmention
last_lookup = if @lookups[post.url]
@lookups[post.url]
elsif last_webmention
Date.parse last_webmention.dig("raw", "verified_date")
end

# should we throttle?
if post.respond_to? "date" # Some docs have no date
if last_lookup && WebmentionIO.post_should_be_throttled?(post, post.date, last_lookup)
WebmentionIO.log "info", "Throttling this post."
return
end
end
end

# Get the last id we have in the hash
since_id = last_webmention ? last_webmention.dig("raw", "id") : false
# Get the last id we have in the hash
since_id = last_webmention ? last_webmention.dig("raw", "id") : false

# Gather the URLs
targets = get_webmention_target_urls(post)
# Gather the URLs
targets = get_webmention_target_urls(post)

# execute the API
response = WebmentionIO.get_response assemble_api_params(targets, since_id)
webmentions = response.dig("links")
if webmentions && !webmentions.empty?
WebmentionIO.log "info", "Here’s what we got back:\n\n#{response.inspect}\n\n"
else
WebmentionIO.log "info", "No webmentions found."
end
# execute the API
response = WebmentionIO.get_response assemble_api_params(targets, since_id)
webmentions = response.dig("links")
if webmentions && !webmentions.empty?
WebmentionIO.log "info", "Here’s what we got back:\n\n#{response.inspect}\n\n"
else
WebmentionIO.log "info", "No webmentions found."
end

@lookups[post.url] = Date.today
cache_new_webmentions(post.url, response)
end
@lookups[post.url] = Date.today
cache_new_webmentions(post.url, response)
end

def get_webmention_target_urls(post)
targets = []
uri = File.join(@site_url, post.url)
targets.push(uri)
def get_webmention_target_urls(post)
targets = []
uri = File.join(@site_url, post.url)
targets.push(uri)

# Redirection?
gather_redirected_targets(post, uri, targets)
# Redirection?
gather_redirected_targets(post, uri, targets)

# Domain changed?
gather_legacy_targets(uri, targets)
# Domain changed?
gather_legacy_targets(uri, targets)

targets
end
targets
end

def gather_redirected_targets(post, uri, targets)
redirected = false
if post.data.key? "redirect_from"
if post.data["redirect_from"].is_a? String
redirected = uri.sub post.url, post.data["redirect_from"]
targets.push(redirected)
elsif post.data["redirect_from"].is_a? Array
post.data["redirect_from"].each do |redirect|
redirected = uri.sub post.url, redirect
def gather_redirected_targets(post, uri, targets)
redirected = false
if post.data.key? "redirect_from"
if post.data["redirect_from"].is_a? String
redirected = uri.sub post.url, post.data["redirect_from"]
targets.push(redirected)
elsif post.data["redirect_from"].is_a? Array
post.data["redirect_from"].each do |redirect|
redirected = uri.sub post.url, redirect
targets.push(redirected)
end
end
end
end
end

def gather_legacy_targets(uri, targets)
if WebmentionIO.config.key? "legacy_domains"
WebmentionIO.log "info", "adding legacy URIs"
WebmentionIO.config["legacy_domains"].each do |domain|
legacy = uri.sub(@site_url, domain)
WebmentionIO.log "info", "adding URI #{legacy}"
targets.push(legacy)
def gather_legacy_targets(uri, targets)
if WebmentionIO.config.key? "legacy_domains"
WebmentionIO.log "info", "adding legacy URIs"
WebmentionIO.config["legacy_domains"].each do |domain|
legacy = uri.sub(@site_url, domain)
WebmentionIO.log "info", "adding URI #{legacy}"
targets.push(legacy)
end
end
end
end

def assemble_api_params(targets, since_id)
api_params = targets.collect { |v| "target[]=#{v}" }.join("&")
api_params << "&since_id=#{since_id}" if since_id
api_params << "&sort-by=published"
api_params
end

def cache_new_webmentions(post_uri, response)
# Get cached webmentions
webmentions = if @cached_webmentions.key? post_uri
@cached_webmentions[post_uri]
else
{}
end

if response && response["links"]
response["links"].reverse_each do |link|
webmention = WebmentionIO::Webmention.new(link, @site)

# Do we already have it?
if webmentions.key? webmention.id
next
end
def assemble_api_params(targets, since_id)
api_params = targets.collect { |v| "target[]=#{v}" }.join("&")
api_params << "&since_id=#{since_id}" if since_id
api_params << "&sort-by=published"
api_params
end

# Add it to the list
WebmentionIO.log "info", webmention.to_hash.inspect
webmentions[webmention.id] = webmention.to_hash
end # each link
end # if response
@cached_webmentions[post_uri] = webmentions
end # process_webmentions
def cache_new_webmentions(post_uri, response)
# Get cached webmentions
webmentions = if @cached_webmentions.key? post_uri
@cached_webmentions[post_uri]
else
{}
end

if response && response["links"]
response["links"].reverse_each do |link|
webmention = WebmentionIO::Webmention.new(link, @site)

# Do we already have it?
if webmentions.key? webmention.id
next
end

# Add it to the list
WebmentionIO.log "info", webmention.to_hash.inspect
webmentions[webmention.id] = webmention.to_hash
end # each link
end # if response
@cached_webmentions[post_uri] = webmentions
end # process_webmentions
end
end
end
Loading