Skip to content

Commit

Permalink
Merge pull request #327 from stephenc/expose-rate-limit-headers
Browse files Browse the repository at this point in the history 
Expose Rate Limit Headers
  • Loading branch information
@kohsuke
kohsuke authored Jan 9, 2017
2 parents 470da06 + 6fcddf4 commit 1266dcc
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 25 deletions.
88 changes: 70 additions & 18 deletions src/main/java/org/kohsuke/github/GitHub.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,10 @@
*/
package org.kohsuke.github;

import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static java.util.logging.Level.FINE;
import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
import static org.kohsuke.github.Previews.DRAX;

import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.VisibilityChecker.Std;
import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
Expand All @@ -49,17 +47,19 @@
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;

import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import org.apache.commons.codec.Charsets;
import org.apache.commons.codec.binary.Base64;

import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.VisibilityChecker.Std;
import com.infradna.tool.bridge_method_injector.WithBridgeMethods;

import javax.annotation.Nonnull;
import java.util.logging.Logger;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.ANY;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
import static java.util.logging.Level.FINE;
import static org.kohsuke.github.Previews.DRAX;

/**
* Root of the GitHub API.
Expand Down Expand Up @@ -90,6 +90,10 @@ public class GitHub {

private HttpConnector connector = HttpConnector.DEFAULT;

private final Object headerRateLimitLock = new Object();
private GHRateLimit headerRateLimit = null;
private volatile GHRateLimit rateLimit = null;

/**
* Creates a client API root object.
*
Expand Down Expand Up @@ -254,6 +258,10 @@ public HttpConnector getConnector() {
return connector;
}

public String getApiUrl() {
return apiUrl;
}

/**
* Sets the custom connector used to make requests to GitHub.
*/
Expand Down Expand Up @@ -287,17 +295,61 @@ public void setConnector(HttpConnector connector) {
*/
public GHRateLimit getRateLimit() throws IOException {
try {
return retrieve().to("/rate_limit", JsonRateLimit.class).rate;
return rateLimit = retrieve().to("/rate_limit", JsonRateLimit.class).rate;
} catch (FileNotFoundException e) {
// GitHub Enterprise doesn't have the rate limit, so in that case
// return some big number that's not too big.
// see issue #78
GHRateLimit r = new GHRateLimit();
r.limit = r.remaining = 1000000;
long hours = 1000L * 60 * 60;
r.reset = new Date(System.currentTimeMillis() + 1 * hours );
return r;
long hour = 60L * 60L; // this is madness, storing the date as seconds in a Date object
r.reset = new Date((System.currentTimeMillis() + hour) / 1000L );
return rateLimit = r;
}
}

/*package*/ void updateRateLimit(@Nonnull GHRateLimit observed) {
synchronized (headerRateLimitLock) {
if (headerRateLimit == null
|| headerRateLimit.getResetDate().getTime() < observed.getResetDate().getTime()
|| headerRateLimit.remaining > observed.remaining) {
headerRateLimit = observed;
LOGGER.log(Level.INFO, "Rate limit now: {0}", headerRateLimit);
}
}
}

/**
* Returns the most recently observed rate limit data or {@code null} if either there is no rate limit
* (for example GitHub Enterprise) or if no requests have been made.
*
* @return the most recently observed rate limit data or {@code null}.
*/
@CheckForNull
public GHRateLimit lastRateLimit() {
synchronized (headerRateLimitLock) {
return headerRateLimit;
}
}

/**
* Gets the current rate limit while trying not to actually make any remote requests unless absolutely necessary.
*
* @return the current rate limit data.
* @throws IOException if we couldn't get the current rate limit data.
*/
@Nonnull
public GHRateLimit rateLimit() throws IOException {
synchronized (headerRateLimitLock) {
if (headerRateLimit != null) {
return headerRateLimit;
}
}
GHRateLimit rateLimit = this.rateLimit;
if (rateLimit == null || rateLimit.getResetDate().getTime() < System.currentTimeMillis()) {
rateLimit = getRateLimit();
}
return rateLimit;
}

/**
Expand Down
76 changes: 69 additions & 7 deletions src/main/java/org/kohsuke/github/Requester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@

import com.fasterxml.jackson.databind.JsonMappingException;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.io.IOUtils;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -42,23 +40,26 @@
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import javax.annotation.WillClose;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;

import static java.util.Arrays.asList;
import static java.util.logging.Level.FINE;
import static org.kohsuke.github.GitHub.*;
import static org.kohsuke.github.GitHub.MAPPER;

/**
* A builder pattern for making HTTP call and parsing its output.
Expand Down Expand Up @@ -281,6 +282,8 @@ private <T> T _to(String tailApiUrl, Class<T> type, T instance) throws IOExcepti
return result;
} catch (IOException e) {
handleApiError(e);
} finally {
noteRateLimit(tailApiUrl);
}
}
}
Expand All @@ -299,6 +302,8 @@ public int asHttpStatusCode(String tailApiUrl) throws IOException {
return uc.getResponseCode();
} catch (IOException e) {
handleApiError(e);
} finally {
noteRateLimit(tailApiUrl);
}
}
}
Expand All @@ -313,6 +318,59 @@ public InputStream asStream(String tailApiUrl) throws IOException {
return wrapStream(uc.getInputStream());
} catch (IOException e) {
handleApiError(e);
} finally {
noteRateLimit(tailApiUrl);
}
}
}

private void noteRateLimit(String tailApiUrl) {
if ("/rate_limit".equals(tailApiUrl)) {
// the rate_limit API is "free"
return;
}
if (tailApiUrl.startsWith("/search")) {
// the search API uses a different rate limit
return;
}
String limit = uc.getHeaderField("X-RateLimit-Limit");
if (StringUtils.isBlank(limit)) {
// if we are missing a header, return fast
return;
}
String remaining = uc.getHeaderField("X-RateLimit-Remaining");
if (StringUtils.isBlank(remaining)) {
// if we are missing a header, return fast
return;
}
String reset = uc.getHeaderField("X-RateLimit-Reset");
if (StringUtils.isBlank(reset)) {
// if we are missing a header, return fast
return;
}
GHRateLimit observed = new GHRateLimit();
try {
observed.limit = Integer.parseInt(limit);
} catch (NumberFormatException e) {
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.log(Level.FINEST, "Malformed X-RateLimit-Limit header value " + limit, e);
}
return;
}
try {
observed.remaining = Integer.parseInt(remaining);
} catch (NumberFormatException e) {
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.log(Level.FINEST, "Malformed X-RateLimit-Remaining header value " + remaining, e);
}
return;
}
try {
observed.reset = new Date(Long.parseLong(reset)); // this is madness, storing the date as seconds
root.updateRateLimit(observed);
} catch (NumberFormatException e) {
if (LOGGER.isLoggable(Level.FINEST)) {
LOGGER.log(Level.FINEST, "Malformed X-RateLimit-Reset header value " + reset, e);
}
}
}
Expand Down Expand Up @@ -382,7 +440,7 @@ private boolean isMethodWithBody() {
}

try {
return new PagingIterator<T>(type, root.getApiURL(s.toString()));
return new PagingIterator<T>(type, tailApiUrl, root.getApiURL(s.toString()));
} catch (IOException e) {
throw new Error(e);
}
Expand All @@ -391,6 +449,7 @@ private boolean isMethodWithBody() {
class PagingIterator<T> implements Iterator<T> {

private final Class<T> type;
private final String tailApiUrl;

/**
* The next batch to be returned from {@link #next()}.
Expand All @@ -402,9 +461,10 @@ class PagingIterator<T> implements Iterator<T> {
*/
private URL url;

PagingIterator(Class<T> type, URL url) {
this.url = url;
PagingIterator(Class<T> type, String tailApiUrl, URL url) {
this.type = type;
this.tailApiUrl = tailApiUrl;
this.url = url;
}

public boolean hasNext() {
Expand Down Expand Up @@ -438,6 +498,8 @@ private void fetch() {
return;
} catch (IOException e) {
handleApiError(e);
} finally {
noteRateLimit(tailApiUrl);
}
}
} catch (IOException e) {
Expand Down

0 comments on commit 1266dcc

Please sign in to comment.