From 9bef7e5cd810088a365332d9ace5cafcf288d664 Mon Sep 17 00:00:00 2001 From: Nikolas Falco Date: Thu, 28 Nov 2024 00:35:03 +0100 Subject: [PATCH] [JENKINS-644418] Add exponential backoff to BitBucket rate limit retry loop Configure Apache HTTP client to use an exponential backoff retry strategy --- .../client/BitbucketCloudApiClient.java | 267 ++---------- .../internal/api/AbstractBitbucketApi.java | 227 +++++++++- .../client/BitbucketServerAPIClient.java | 404 +++++------------- .../BitbucketIntegrationClientFactory.java | 17 +- .../client/BitbucketServerAPIClientTest.java | 59 ++- ...wse-Jenkinsfile_at_feature_2Fpipeline.json | 3 + ...der-Jenkinsfile_at_feature_2Fpipeline.json | 3 + 7 files changed, 401 insertions(+), 579 deletions(-) create mode 100644 src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json create mode 100644 src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java index 77458316c..8510188d8 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java @@ -33,7 +33,6 @@ import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketPullRequest; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRepository; -import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketTeam; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook; import com.cloudbees.jenkins.plugins.bitbucket.api.credentials.BitbucketUsernamePasswordAuthenticator; @@ -64,9 +63,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -76,33 +72,14 @@ import java.util.logging.Level; import javax.imageio.ImageIO; import jenkins.scm.api.SCMFile; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.SocketConfig; import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.ProtectedExternally; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -116,21 +93,10 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit // Limit images to 16k private static final int MAX_AVATAR_LENGTH = 16384; private static final int MAX_PAGE_LENGTH = 100; - private static final HttpClientConnectionManager connectionManager = getConnectionManager(); - private CloseableHttpClient client; - private final String owner; - private final String projectKey; - private final String repositoryName; - private final boolean enableCache; - private final BitbucketAuthenticator authenticator; - private static final Cache cachedTeam = new Cache<>(6, HOURS); - private static final Cache cachedAvatar = new Cache<>(6, HOURS); - private static final Cache> cachedRepositories = new Cache<>(3, HOURS); - private static final Cache cachedCommits = new Cache<>(24, HOURS); - private transient BitbucketRepository cachedRepository; - private transient String cachedDefaultBranch; - private static HttpClientConnectionManager getConnectionManager() { + protected static final HttpClientConnectionManager connectionManager = connectionManager(); + + private static HttpClientConnectionManager connectionManager() { try { PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); // NOSONAR connManager.setDefaultMaxPerRoute(20); @@ -143,6 +109,18 @@ private static HttpClientConnectionManager getConnectionManager() { } } + private final CloseableHttpClient client; + private final String owner; + private final String projectKey; + private final String repositoryName; + private final boolean enableCache; + private static final Cache cachedTeam = new Cache<>(6, HOURS); + private static final Cache cachedAvatar = new Cache<>(6, HOURS); + private static final Cache> cachedRepositories = new Cache<>(3, HOURS); + private static final Cache cachedCommits = new Cache<>(24, HOURS); + private transient BitbucketRepository cachedRepository; + private transient String cachedDefaultBranch; + public static List stats() { List stats = new ArrayList<>(); stats.add("Team: " + cachedTeam.stats().toString()); @@ -166,7 +144,7 @@ public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int r public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int repositoriesCacheDuration, String owner, String projectKey, String repositoryName, BitbucketAuthenticator authenticator) { - this.authenticator = authenticator; + super(authenticator); this.owner = owner; this.projectKey = projectKey; this.repositoryName = repositoryName; @@ -175,32 +153,7 @@ public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int r cachedTeam.setExpireDuration(teamCacheDuration, MINUTES); cachedRepositories.setExpireDuration(repositoriesCacheDuration, MINUTES); } - - // Create Http client - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() - .setConnectionManager(connectionManager) - .setConnectionManagerShared(connectionManager != null) - .setRetryHandler(new StandardHttpRequestRetryHandler()); - - if (authenticator != null) { - authenticator.configureBuilder(httpClientBuilder); - - context = HttpClientContext.create(); - authenticator.configureContext(context, API_HOST); - } - - setClientProxyParams("bitbucket.org", httpClientBuilder); - - this.client = httpClientBuilder.build(); - } - - @Override - protected void finalize() throws Throwable { - if (client != null) { - client.close(); - } - - super.finalize(); + this.client = super.buildClient("bitbucket.org"); } /** @@ -311,8 +264,8 @@ private void setupClosureForPRBranch(BitbucketPullRequestValue pullRequest) { @Deprecated @CheckForNull public String getLogin() { - if (authenticator != null) { - return authenticator.getId(); + if (getAuthenticator() != null) { + return getAuthenticator().getId(); } return null; } @@ -757,7 +710,7 @@ public AvatarImage getTeamAvatar() throws IOException, InterruptedException { /** * The role parameter only makes sense when the request is authenticated, so - * if there is no auth information ({@link #authenticator}) the role will be omitted. + * if there is no auth information ({@link #getAuthenticator()}) the role will be omitted. */ @NonNull @Override @@ -766,8 +719,8 @@ public List getRepositories(@CheckForNull UserRoleInRe StringBuilder cacheKey = new StringBuilder(); cacheKey.append(owner); - if (authenticator != null) { - cacheKey.append("::").append(authenticator.getId()); + if (getAuthenticator() != null) { + cacheKey.append("::").append(getAuthenticator().getId()); } else { cacheKey.append("::"); } @@ -781,7 +734,7 @@ public List getRepositories(@CheckForNull UserRoleInRe } else { cacheKey.append("::"); } - if (role != null && authenticator != null) { + if (role != null && getAuthenticator() != null) { template.set("role", role.getId()); cacheKey.append("::").append(role.getId()); } else { @@ -824,94 +777,7 @@ public List getRepositories() throws IOException, Inte return getRepositories(null); } - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethod(HttpRequestBase httpMethod) throws InterruptedException, IOException { - return executeMethod(API_HOST, httpMethod); - } - - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws InterruptedException, IOException { - HttpClientContext requestContext = null; - if (API_HOST.equals(host)) { - requestContext = context; - if (authenticator != null) { - authenticator.configureRequest(httpMethod); - } - } - - RequestConfig.Builder requestConfig = RequestConfig.custom(); - String connectTimeout = System.getProperty("http.connect.timeout", "10"); - requestConfig.setConnectTimeout(Integer.parseInt(connectTimeout) * 1000); - String connectionRequestTimeout = System.getProperty("http.connect.request.timeout", "60"); - requestConfig.setConnectionRequestTimeout(Integer.parseInt(connectionRequestTimeout) * 1000); - String socketTimeout = System.getProperty("http.socket.timeout", "60"); - requestConfig.setSocketTimeout(Integer.parseInt(socketTimeout) * 1000); - httpMethod.setConfig(requestConfig.build()); - - CloseableHttpResponse response = client.execute(host, httpMethod, requestContext); - while (response.getStatusLine().getStatusCode() == API_RATE_LIMIT_STATUS_CODE) { - release(httpMethod); - if (Thread.interrupted()) { - throw new InterruptedException(); - } - /* - TODO: When bitbucket starts supporting rate limit expiration time, remove 5 sec wait and put code - to wait till expiration time is over. It should also fix the wait for ever loop. - */ - logger.fine("Bitbucket Cloud API rate limit reached, sleeping for 5 sec then retry..."); - Thread.sleep(5000); - response = client.execute(host, httpMethod, requestContext); - } - return response; - } - - /** - * Caller's responsible to close the InputStream. - */ - private InputStream getRequestAsInputStream(String path) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(path); - HttpHost host = null; - - // Extract host from URL, if present - try { - URI uri = new URI(path); - if (uri.isAbsolute() && ! uri.isOpaque()) { - host = HttpHost.create(""+uri.getScheme()+"://"+uri.getAuthority()); - } - } catch (URISyntaxException ex) { - } - // Use default API Host otherwise - if (host == null) { - host = API_HOST; - } - - try { - CloseableHttpResponse response = executeMethod(host, httpget); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NOT_FOUND) { - EntityUtils.consume(response.getEntity()); - response.close(); - throw new FileNotFoundException("URL: " + path); - } - if (statusCode != HttpStatus.SC_OK) { - String content = getResponseContent(response); - throw buildResponseException(response, content); - } - return new ClosingConnectionInputStream(response, httpget, connectionManager); - } catch (BitbucketRequestException | FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } - } - - private String getRequest(String path) throws IOException, InterruptedException { - try (InputStream inputStream = getRequestAsInputStream(path)){ - return IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - } - - private BufferedImage getImageRequest(String path) throws IOException, InterruptedException { + private BufferedImage getImageRequest(String path) throws IOException { try (InputStream inputStream = getRequestAsInputStream(path)) { int length = MAX_AVATAR_LENGTH; BufferedInputStream bis = new BufferedInputStream(inputStream, length); @@ -919,85 +785,24 @@ private BufferedImage getImageRequest(String path) throws IOException, Interrupt } } - private int headRequestStatus(String path) throws IOException, InterruptedException { - HttpHead httpHead = new HttpHead(path); - try(CloseableHttpResponse response = executeMethod(httpHead)) { - EntityUtils.consume(response.getEntity()); - return response.getStatusLine().getStatusCode(); - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - release(httpHead); - } - } - - private void deleteRequest(String path) throws IOException, InterruptedException { - HttpDelete httppost = new HttpDelete(path); - try(CloseableHttpResponse response = executeMethod(httppost)) { - EntityUtils.consume(response.getEntity()); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NOT_FOUND) { - throw new FileNotFoundException("URL: " + path); - } - if (statusCode != HttpStatus.SC_NO_CONTENT) { - throw buildResponseException(response, getResponseContent(response)); - } - } catch (BitbucketRequestException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - release(httppost); - } - } - - private String doRequest(HttpRequestBase request) throws IOException, InterruptedException { - try(CloseableHttpResponse response = executeMethod(request)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NO_CONTENT) { - EntityUtils.consume(response.getEntity()); - // 204, no content - return ""; - } - String content = getResponseContent(response); - EntityUtils.consume(response.getEntity()); - if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { - throw buildResponseException(response, content); - } - return content; - } catch (BitbucketRequestException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + request, e); - } finally { - release(request); - } - } - - private void release(HttpRequestBase method) { - method.releaseConnection(); - connectionManager.closeExpiredConnections(); - } - - private String putRequest(String path, String content) throws IOException, InterruptedException { - HttpPut request = new HttpPut(path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(request); + @Override + protected HttpClientConnectionManager getConnectionManager() { + return connectionManager; } - private String postRequest(String path, String content) throws IOException, InterruptedException { - HttpPost httppost = new HttpPost(path); - httppost.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(httppost); + @NonNull + @Override + protected HttpHost getHost() { + return API_HOST; } - private String postRequest(String path, List params) throws IOException, InterruptedException { - HttpPost httppost = new HttpPost(path); - httppost.setEntity(new UrlEncodedFormEntity(params)); - return doRequest(httppost); + @NonNull + @Override + protected CloseableHttpClient getClient() { + return client; } - private List getAllBranches(String response) throws IOException, InterruptedException { + private List getAllBranches(String response) throws IOException { List branches = new ArrayList<>(); BitbucketCloudPage page = JsonParser.mapper.readValue(response, new TypeReference>(){}); diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/internal/api/AbstractBitbucketApi.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/internal/api/AbstractBitbucketApi.java index 4b02d9103..37c75f88b 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/internal/api/AbstractBitbucketApi.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/internal/api/AbstractBitbucketApi.java @@ -23,39 +23,75 @@ */ package com.cloudbees.jenkins.plugins.bitbucket.internal.api; -import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; -import edu.umd.cs.findbugs.annotations.CheckForNull; -import hudson.ProxyConfiguration; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.net.Proxy; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.logging.Logger; -import jenkins.model.Jenkins; + import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; +import org.apache.http.client.BackoffManager; import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.AIMDBackoffManager; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultBackoffStrategy; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.pool.ConnPoolControl; +import org.apache.http.util.EntityUtils; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.ProtectedExternally; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticator; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; +import com.cloudbees.jenkins.plugins.bitbucket.client.ClosingConnectionInputStream; + +import edu.umd.cs.findbugs.annotations.CheckForNull; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import hudson.ProxyConfiguration; +import jenkins.model.Jenkins; + @Restricted(ProtectedExternally.class) public abstract class AbstractBitbucketApi { protected static final int API_RATE_LIMIT_STATUS_CODE = 429; protected final Logger logger = Logger.getLogger(this.getClass().getName()); - protected HttpClientContext context; + private final BitbucketAuthenticator authenticator; + private HttpClientContext context; + + protected AbstractBitbucketApi(BitbucketAuthenticator authenticator) { + this.authenticator = authenticator; + } protected String truncateMiddle(@CheckForNull String value, int maxLength) { int length = StringUtils.length(value); @@ -108,7 +144,45 @@ private long getLenghtFromHeader(CloseableHttpResponse response) { return len; } - protected void setClientProxyParams(String host, HttpClientBuilder builder) { + @SuppressWarnings("unchecked") + protected CloseableHttpClient buildClient(@Nullable String host) { + int connectTimeout = Integer.getInteger("http.connect.timeout", 10); + int connectionRequestTimeout = Integer.getInteger("http.connect.request.timeout", 60); + int socketTimeout = Integer.getInteger("http.socket.timeout", 60); + + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(connectTimeout * 1000) + .setConnectionRequestTimeout(connectionRequestTimeout * 1000) + .setSocketTimeout(socketTimeout * 1000) + .build(); + + HttpClientConnectionManager connectionManager = getConnectionManager(); + BackoffManager backoffManager = null; + if (connectionManager instanceof ConnPoolControl connPerRoute) { + backoffManager = new AIMDBackoffManager(connPerRoute); + } + + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .useSystemProperties() + .setConnectionManager(connectionManager) + .setConnectionManagerShared(connectionManager != null) + .setConnectionBackoffStrategy(new DefaultBackoffStrategy()) + .setBackoffManager(backoffManager) + .setRetryHandler(new StandardHttpRequestRetryHandler()) + .setDefaultRequestConfig(config) + .disableCookieManagement(); + + if (authenticator != null) { + authenticator.configureBuilder(httpClientBuilder); + + context = HttpClientContext.create(); + authenticator.configureContext(context, getHost()); + } + setClientProxyParams(host, httpClientBuilder); + return httpClientBuilder.build(); + } + + private void setClientProxyParams(String host, HttpClientBuilder builder) { Jenkins jenkins = Jenkins.getInstanceOrNull(); // because unit test ProxyConfiguration proxyConfig = jenkins != null ? jenkins.proxy : null; @@ -149,4 +223,145 @@ protected void setClientProxyParams(String host, HttpClientBuilder builder) { } } + @Nullable + protected abstract HttpClientConnectionManager getConnectionManager(); + + @NonNull + protected abstract HttpHost getHost(); + + @NonNull + protected abstract CloseableHttpClient getClient(); + + /* for test purpose */ + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { + if (authenticator != null) { + authenticator.configureRequest(httpMethod); + } + return getClient().execute(host, httpMethod, context); + } + + protected String doRequest(HttpRequestBase request) throws IOException { + try (CloseableHttpResponse response = executeMethod(getHost(), request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_NOT_FOUND) { + throw new FileNotFoundException("URL: " + request.getURI()); + } + if (statusCode == HttpStatus.SC_NO_CONTENT) { + EntityUtils.consume(response.getEntity()); + // 204, no content + return ""; + } + String content = getResponseContent(response); + EntityUtils.consume(response.getEntity()); + if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { + throw buildResponseException(response, content); + } + return content; + } catch (BitbucketRequestException e) { + throw e; + } catch (IOException e) { + throw new IOException("Communication error for url: " + request, e); + } finally { + release(request); + } + } + + private void release(HttpRequestBase method) { + method.releaseConnection(); + HttpClientConnectionManager connectionManager = getConnectionManager(); + if (connectionManager != null) { + connectionManager.closeExpiredConnections(); + } + } + + /** + * Caller's responsible to close the InputStream. + */ + protected InputStream getRequestAsInputStream(String path) throws IOException { + HttpGet httpget = new HttpGet(path); + HttpHost host = getHost(); + + // Extract host from URL, if present + try { + URI uri = new URI(host.toURI()); + if (uri.isAbsolute() && ! uri.isOpaque()) { + host = HttpHost.create(uri.getScheme() + "://" + uri.getAuthority()); + } + } catch (URISyntaxException ex) { + // use default + } + + try (CloseableHttpResponse response = executeMethod(host, httpget)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_NOT_FOUND) { + EntityUtils.consume(response.getEntity()); + throw new FileNotFoundException("URL: " + path); + } + if (statusCode != HttpStatus.SC_OK) { + String content = getResponseContent(response); + throw buildResponseException(response, content); + } + return new ClosingConnectionInputStream(response, httpget, getConnectionManager()); + } catch (BitbucketRequestException | FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new IOException("Communication error for url: " + path, e); + } finally { + release(httpget); + } + } + + protected int headRequestStatus(String path) throws IOException { + HttpHead httpHead = new HttpHead(path); + try (CloseableHttpResponse response = executeMethod(getHost(), httpHead)) { + EntityUtils.consume(response.getEntity()); + return response.getStatusLine().getStatusCode(); + } catch (IOException e) { + throw new IOException("Communication error for url: " + path, e); + } finally { + release(httpHead); + } + } + + protected String getRequest(String path) throws IOException { + HttpGet httpget = new HttpGet(path); + return doRequest(httpget); + } + + protected String postRequest(String path, List params) throws IOException { + HttpPost request = new HttpPost(path); + request.setEntity(new UrlEncodedFormEntity(params)); + return doRequest(request); + } + + protected String postRequest(String path, String content) throws IOException { + HttpPost request = new HttpPost(path); + request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); + return doRequest(request); + } + + protected String putRequest(String path, String content) throws IOException { + HttpPut request = new HttpPut(path); + request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); + return doRequest(request); + } + + protected String deleteRequest(String path) throws IOException { + HttpDelete request = new HttpDelete(path); + return doRequest(request); + } + + // TODO move interface to autoclosable + @Override + protected void finalize() throws Throwable { + if (getClient() != null) { + getClient().close(); + } + + super.finalize(); + } + + protected BitbucketAuthenticator getAuthenticator() { + return authenticator; + } } diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java index 7802021b7..742cb05ec 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java @@ -23,6 +23,36 @@ */ package com.cloudbees.jenkins.plugins.bitbucket.server.client; +import static java.util.Objects.requireNonNull; + +import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.function.Predicate; +import java.util.logging.Level; + +import javax.imageio.ImageIO; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; + import com.cloudbees.jenkins.plugins.bitbucket.JsonParser; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketApi; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticator; @@ -62,55 +92,12 @@ import com.damnhandy.uri.template.UriTemplate; import com.damnhandy.uri.template.impl.Operator; import com.fasterxml.jackson.core.type.TypeReference; + import edu.umd.cs.findbugs.annotations.CheckForNull; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import hudson.Main; import hudson.Util; -import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.function.Predicate; -import java.util.logging.Level; -import javax.imageio.ImageIO; import jenkins.scm.api.SCMFile; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.ProtectedExternally; - -import static java.util.Objects.requireNonNull; /** * Bitbucket API client. @@ -119,7 +106,7 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements BitbucketApi { // Max avatar image length in bytes - private static final int MAX_AVATAR_SIZE = 16384; + private static final int MAX_AVATAR_LENGTH = 16384; private static final String API_BASE_PATH = "/rest/api/1.0"; private static final String API_REPOSITORIES_PATH = API_BASE_PATH + "/projects/{owner}/repos{?start,limit}"; @@ -133,7 +120,7 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements Bi private static final String API_PULL_REQUEST_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}"; private static final String API_PULL_REQUEST_MERGE_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}/merge"; private static final String API_PULL_REQUEST_CHANGES_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}/changes{?start,limit}"; - static final String API_BROWSE_PATH = API_REPOSITORY_PATH + "/browse{/path*}{?at}"; + private static final String API_BROWSE_PATH = API_REPOSITORY_PATH + "/browse{/path*}{?at}"; private static final String API_COMMITS_PATH = API_REPOSITORY_PATH + "/commits{/hash}"; private static final String API_PROJECT_PATH = API_BASE_PATH + "/projects/{owner}"; private static final String AVATAR_PATH = API_BASE_PATH + "/projects/{owner}/avatar.png"; @@ -148,35 +135,37 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements Bi private static final String API_MIRRORS_FOR_REPO_PATH = "/rest/mirroring/1.0/repos/{id}/mirrors"; private static final String API_MIRRORS_PATH = "/rest/mirroring/1.0/mirrorServers"; - private static final Integer DEFAULT_PAGE_LIMIT = 200; - private static final Duration API_RATE_LIMIT_INITIAL_SLEEP = Main.isUnitTest ? Duration.ofMillis(100) : Duration.ofSeconds(5); - private static final Duration API_RATE_LIMIT_MAX_SLEEP = Duration.ofMinutes(30); + + protected static final HttpClientConnectionManager connectionManager = connectionManager(); + + private static HttpClientConnectionManager connectionManager() { + try { + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); // NOSONAR + connManager.setDefaultMaxPerRoute(20); + connManager.setMaxTotal(22); + return connManager; + } catch (Exception e) { + // in case of exception this avoids ClassNotFoundError which prevents the classloader from loading this class again + return null; + } + } /** * Repository owner. */ private final String owner; - /** * The repository that this object is managing. */ private final String repositoryName; - /** * Indicates if the client is using user-centric API endpoints or project API otherwise. */ private final boolean userCentric; - - /** - * Credentials to access API services. - * Almost @NonNull (but null is accepted for anonymous access). - */ - private final BitbucketAuthenticator authenticator; - private final String baseURL; - private final BitbucketServerWebhookImplementation webhookImplementation; + private final CloseableHttpClient client; @Deprecated public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, @CheckForNull String repositoryName, @@ -193,12 +182,13 @@ public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, @CheckForNull String repositoryName, @CheckForNull BitbucketAuthenticator authenticator, boolean userCentric, @NonNull BitbucketServerWebhookImplementation webhookImplementation) { - this.authenticator = authenticator; + super(authenticator); this.userCentric = userCentric; this.owner = owner; this.repositoryName = repositoryName; this.baseURL = Util.removeTrailingSlash(baseURL); this.webhookImplementation = requireNonNull(webhookImplementation); + this.client = super.buildClient(baseURL); } /** @@ -240,7 +230,7 @@ public String getRepositoryName() { @Override public List getPullRequests() throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName); return getPullRequests(template); @@ -249,7 +239,7 @@ public List getPullRequests() throws IOException, In @NonNull public List getOutgoingOpenPullRequests(String fromRef) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("at", fromRef) @@ -261,7 +251,7 @@ public List getOutgoingOpenPullRequests(String fromR @NonNull public List getIncomingOpenPullRequests(String toRef) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("at", toRef) @@ -368,7 +358,7 @@ private void setupClosureForPRBranch(BitbucketServerPullRequest pr) { private void callPullRequestChangesById(@NonNull String id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_CHANGES_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_CHANGES_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id).set("limit", 1) @@ -378,7 +368,7 @@ private void callPullRequestChangesById(@NonNull String id) throws IOException, private boolean getPullRequestCanMergeById(@NonNull String id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_MERGE_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_MERGE_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id) @@ -398,7 +388,7 @@ private boolean getPullRequestCanMergeById(@NonNull String id) throws IOExceptio @NonNull public BitbucketPullRequest getPullRequestById(@NonNull Integer id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id) @@ -426,7 +416,7 @@ public BitbucketRepository getRepository() throws IOException, InterruptedExcept "Cannot get a repository from an API instance that is not associated with a repository"); } String url = UriTemplate - .fromTemplate(API_REPOSITORY_PATH) + .fromTemplate(this.baseURL + API_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -448,7 +438,7 @@ public BitbucketRepository getRepository() throws IOException, InterruptedExcept @NonNull public List getMirrors() throws IOException, InterruptedException { UriTemplate uriTemplate = UriTemplate - .fromTemplate(API_MIRRORS_PATH); + .fromTemplate(this.baseURL + API_MIRRORS_PATH); return getResources(uriTemplate, BitbucketMirrorServerDescriptors.class); } @@ -462,7 +452,7 @@ public List getMirrors() throws IOException, InterruptedE @NonNull public List getMirrors(@NonNull Long repositoryId) throws IOException, InterruptedException { UriTemplate uriTemplate = UriTemplate - .fromTemplate(API_MIRRORS_FOR_REPO_PATH) + .fromTemplate(this.baseURL + API_MIRRORS_FOR_REPO_PATH) .set("id", repositoryId); return getResources(uriTemplate, BitbucketMirroredRepositoryDescriptors.class); } @@ -477,8 +467,7 @@ public List getMirrors(@NonNull Long repo */ @NonNull public BitbucketMirroredRepository getMirroredRepository(@NonNull String url) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(url); - String response = getRequest(httpget); + String response = getRequest(url); try { return JsonParser.toJava(response, BitbucketMirroredRepository.class); } catch (IOException e) { @@ -493,7 +482,7 @@ public BitbucketMirroredRepository getMirroredRepository(@NonNull String url) th public void postCommitComment(@NonNull String hash, @NonNull String comment) throws IOException, InterruptedException { postRequest( UriTemplate - .fromTemplate(API_COMMIT_COMMENT_PATH) + .fromTemplate(this.baseURL + API_COMMIT_COMMENT_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("hash", hash) @@ -512,7 +501,7 @@ public void postBuildStatus(@NonNull BitbucketBuildStatus status) throws IOExcep BitbucketBuildStatus newStatus = new BitbucketBuildStatus(status); newStatus.setName(truncateMiddle(newStatus.getName(), 255)); - String url = UriTemplate.fromTemplate(API_COMMIT_STATUS_PATH) + String url = UriTemplate.fromTemplate(this.baseURL + API_COMMIT_STATUS_PATH) .set("hash", newStatus.getHash()) .expand(); postRequest(url, JsonParser.toJson(newStatus)); @@ -524,13 +513,13 @@ public void postBuildStatus(@NonNull BitbucketBuildStatus status) throws IOExcep @Override public boolean checkPathExists(@NonNull String branchOrHash, @NonNull String path) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_BROWSE_PATH) + .fromTemplate(this.baseURL + API_BROWSE_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", path.split(Operator.PATH.getSeparator())) .set("at", branchOrHash) .expand(); - int status = getRequestStatus(url); + int status = headRequestStatus(url); if (HttpStatus.SC_OK == status) { return true; // Bitbucket returns UNAUTHORIZED when no credentials are provided @@ -546,7 +535,7 @@ public boolean checkPathExists(@NonNull String branchOrHash, @NonNull String pat @Override public String getDefaultBranch() throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_DEFAULT_BRANCH_PATH) + .fromTemplate(this.baseURL + API_DEFAULT_BRANCH_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -576,7 +565,7 @@ public BitbucketServerBranch getTag(@NonNull String tagName) throws IOException, @Override @NonNull public List getTags() throws IOException, InterruptedException { - return getServerBranches(API_TAGS_PATH); + return getServerBranches(this.baseURL + API_TAGS_PATH); } /** @@ -593,7 +582,7 @@ public BitbucketServerBranch getBranch(@NonNull String branchName) throws IOExce @Override @NonNull public List getBranches() throws IOException, InterruptedException { - return getServerBranches(API_BRANCHES_PATH); + return getServerBranches(this.baseURL + API_BRANCHES_PATH); } private List getServerBranches(String apiPath) throws IOException, InterruptedException { @@ -614,7 +603,7 @@ private List getServerBranches(String apiPath) throws IOE private BitbucketServerBranch getSingleTag(String tagName) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_TAGS_FILTERED_PATH) + .fromTemplate(this.baseURL + API_TAGS_FILTERED_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("filterText", tagName); @@ -629,7 +618,7 @@ private BitbucketServerBranch getSingleTag(String tagName) throws IOException, I private BitbucketServerBranch getSingleBranch(String branchName) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_BRANCHES_FILTERED_PATH) + .fromTemplate(this.baseURL + API_BRANCHES_FILTERED_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("filterText", branchName); @@ -647,7 +636,7 @@ private BitbucketServerBranch getSingleBranch(String branchName) throws IOExcept @Override public BitbucketCommit resolveCommit(@NonNull String hash) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_COMMITS_PATH) + .fromTemplate(this.baseURL + API_COMMITS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("hash", hash) @@ -679,7 +668,7 @@ public void registerCommitWebHook(BitbucketWebHook hook) throws IOException, Int case PLUGIN: putRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(), @@ -690,7 +679,7 @@ public void registerCommitWebHook(BitbucketWebHook hook) throws IOException, Int case NATIVE: postRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(), @@ -710,7 +699,7 @@ public void updateCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case PLUGIN: postRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_CONFIG_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_CONFIG_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -721,7 +710,7 @@ public void updateCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case NATIVE: putRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -741,7 +730,7 @@ public void removeCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case PLUGIN: deleteRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_CONFIG_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_CONFIG_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -752,7 +741,7 @@ public void removeCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case NATIVE: deleteRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -772,7 +761,7 @@ public List getWebHooks() throws IOException, Interr switch (webhookImplementation) { case PLUGIN: String url = UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -780,7 +769,7 @@ public List getWebHooks() throws IOException, Interr return JsonParser.toJava(response, BitbucketServerWebhooks.class); case NATIVE: UriTemplate urlTemplate = UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName); return getResources(urlTemplate, NativeBitbucketServerWebhooks.class); @@ -797,7 +786,9 @@ public BitbucketTeam getTeam() throws IOException, InterruptedException { if (userCentric) { return null; } else { - String url = UriTemplate.fromTemplate(API_PROJECT_PATH).set("owner", getOwner()).expand(); + String url = UriTemplate.fromTemplate(this.baseURL + API_PROJECT_PATH) + .set("owner", getOwner()) + .expand(); try { String response = getRequest(url); return JsonParser.toJava(response, BitbucketServerProject.class); @@ -817,7 +808,9 @@ public AvatarImage getTeamAvatar() throws IOException { if (userCentric) { return null; } else { - String url = UriTemplate.fromTemplate(AVATAR_PATH).set("owner", getOwner()).expand(); + String url = UriTemplate.fromTemplate(this.baseURL + AVATAR_PATH) + .set("owner", getOwner()) + .expand(); try { BufferedImage response = getImageRequest(url); return new AvatarImage(response, System.currentTimeMillis()); @@ -839,7 +832,7 @@ public AvatarImage getTeamAvatar() throws IOException { public List getRepositories(@CheckForNull UserRoleInRepository role) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_REPOSITORIES_PATH) + .fromTemplate(this.baseURL + API_REPOSITORIES_PATH) .set("owner", getUserCentricOwner()); List repositories; @@ -909,7 +902,7 @@ private V getResource(UriTemplate template, Class V getResource(UriTemplate template, Class 0 ? Math.min(MAX_AVATAR_SIZE, length) : MAX_AVATAR_SIZE; - BufferedInputStream bis = new BufferedInputStream(is, length); - content = ImageIO.read(bis); - } - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) { - throw new FileNotFoundException("URL: " + path); - } - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new BitbucketRequestException(response.getStatusLine().getStatusCode(), - "HTTP request error. Status: " + response.getStatusLine().getStatusCode() + ": " - + response.getStatusLine().getReasonPhrase() + ".\n" + response); - } - return content; - } catch (BitbucketRequestException | FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - httpget.releaseConnection(); - } - - } - - /** - * Create HttpClient from given host/port - * @param request the {@link HttpRequestBase} for which an HttpClient will be created - * @return CloseableHttpClient - */ - private CloseableHttpClient getHttpClient(final HttpRequestBase request) { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - httpClientBuilder.useSystemProperties(); - httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); - httpClientBuilder.disableCookieManagement(); - - RequestConfig.Builder requestConfig = RequestConfig.custom(); - String connectTimeout = System.getProperty("http.connect.timeout", "10"); - requestConfig.setConnectTimeout(Integer.parseInt(connectTimeout) * 1000); - String connectionRequestTimeout = System.getProperty("http.connect.request.timeout", "60"); - requestConfig.setConnectionRequestTimeout(Integer.parseInt(connectionRequestTimeout) * 1000); - String socketTimeout = System.getProperty("http.socket.timeout", "60"); - requestConfig.setSocketTimeout(Integer.parseInt(socketTimeout) * 1000); - request.setConfig(requestConfig.build()); - - final String host = getMethodHost(request); - - if (authenticator != null) { - authenticator.configureBuilder(httpClientBuilder); - - context = HttpClientContext.create(); - authenticator.configureContext(context, HttpHost.create(host)); - } - - setClientProxyParams(host, httpClientBuilder); - - return httpClientBuilder.build(); - } - - private int getRequestStatus(String path) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(this.baseURL + path); - if (authenticator != null) { - authenticator.configureRequest(httpget); - } - - try(CloseableHttpClient client = getHttpClient(httpget); - CloseableHttpResponse response = executeMethod(client, httpget)) { - EntityUtils.consume(response.getEntity()); - return response.getStatusLine().getStatusCode(); - } finally { - httpget.releaseConnection(); + try (InputStream inputStream = getRequestAsInputStream(path)) { + int length = MAX_AVATAR_LENGTH; + BufferedInputStream bis = new BufferedInputStream(inputStream, length); + return ImageIO.read(bis); } } - private static String getMethodHost(HttpRequestBase method) { - URI uri = method.getURI(); - String scheme = uri.getScheme() == null ? "http" : uri.getScheme(); - return scheme + "://" + uri.getAuthority(); - } - - private String postRequest(String path, List params) throws IOException, InterruptedException { - HttpPost request = new HttpPost(this.baseURL + path); - request.setEntity(new UrlEncodedFormEntity(params)); - return postRequest(request); - } - - private String postRequest(String path, String content) throws IOException, InterruptedException { - HttpPost request = new HttpPost(this.baseURL + path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return postRequest(request); + @Override + protected HttpClientConnectionManager getConnectionManager() { + return connectionManager(); } - private String postRequest(HttpPost httppost) throws IOException, InterruptedException { - return doRequest(httppost); + @NonNull + @Override + protected CloseableHttpClient getClient() { + return client; } - private String doRequest(HttpRequestBase request) throws IOException, InterruptedException { - if (authenticator != null) { - authenticator.configureRequest(request); - } - - try (CloseableHttpClient client = getHttpClient(request); - CloseableHttpResponse response = executeMethod(client, request)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NO_CONTENT) { - EntityUtils.consume(response.getEntity()); - // 204, no content - return ""; - } - String content = getResponseContent(response); - EntityUtils.consume(response.getEntity()); - if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { - throw new BitbucketRequestException(statusCode, "HTTP request error. Status: " + statusCode + ": " + response.getStatusLine().getReasonPhrase() + ".\n" + response); + @NonNull + @Override + protected HttpHost getHost() { + String url = baseURL; + try { + // it's really needed? + URL tmp = new URL(baseURL); + if (tmp.getProtocol() == null) { + url = new URL("http", tmp.getHost(), tmp.getPort(), tmp.getFile()).toString(); } - return content; - } finally { - request.releaseConnection(); + } catch (MalformedURLException e) { } - } - - private String putRequest(String path, String content) throws IOException, InterruptedException { - HttpPut request = new HttpPut(this.baseURL + path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(request); - } - - private String deleteRequest(String path) throws IOException, InterruptedException { - HttpDelete request = new HttpDelete(this.baseURL + path); - return doRequest(request); + return HttpHost.create(url); } @Override @@ -1116,7 +963,7 @@ public Iterable getDirectoryContent(BitbucketSCMFile directory) throws int start=0; String branchOrHash = directory.getHash().contains("+") ? directory.getRef() : directory.getHash(); UriTemplate template = UriTemplate - .fromTemplate(API_BROWSE_PATH + "{&start,limit}") + .fromTemplate(this.baseURL + API_BROWSE_PATH + "{&start,limit}") .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", directory.getPath().split(Operator.PATH.getSeparator())) @@ -1165,7 +1012,7 @@ public InputStream getFileContent(BitbucketSCMFile file) throws IOException, Int int start=0; String branchOrHash = file.getHash().contains("+") ? file.getRef() : file.getHash(); UriTemplate template = UriTemplate - .fromTemplate(API_BROWSE_PATH + "{&start,limit}") + .fromTemplate(this.baseURL + API_BROWSE_PATH + "{&start,limit}") .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", file.getPath().split(Operator.PATH.getSeparator())) @@ -1199,35 +1046,4 @@ private Map collectLines(String response, final List line return content; } - private CloseableHttpResponse executeMethod(CloseableHttpClient client, HttpRequestBase httpMethod) throws IOException, InterruptedException { - CloseableHttpResponse response = executeMethodNoRetry(client, httpMethod, context); - Instant start = Instant.now(); - Instant forcedEnd = start.plus(API_RATE_LIMIT_MAX_SLEEP); - Duration sleepDuration = API_RATE_LIMIT_INITIAL_SLEEP; - while (response.getStatusLine().getStatusCode() == API_RATE_LIMIT_STATUS_CODE - && Instant.now().plus(sleepDuration).isBefore(forcedEnd)) { - response.close(); - httpMethod.releaseConnection(); - /* - * TODO: If The Bitbucket Server API ever starts sending rate limit expiration time, we should - * change this to a more precise sleep. - * TODO: It would be better to log this to a context-appropriate TaskListener, e.g. an org/repo scan log. - */ - logger.log(Level.FINE, "Bitbucket server API rate limit reached, sleeping for {0} before retrying", - sleepDuration); - Thread.sleep(sleepDuration.toMillis()); - // Duration increases exponentially: 5s, 7s, 10s, 15s, 22s, ... 6m6s, 9m9s. - // We will retry at most 13 times and sleep for roughly 27 minutes. - sleepDuration = Duration.ofSeconds((int)(sleepDuration.getSeconds() * 1.5)); - response = executeMethodNoRetry(client, httpMethod, context); - } - return response; - } - - // Exists just so it can be mocked in BitbucketIntegrationClientFactory. - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethodNoRetry(CloseableHttpClient client, HttpRequestBase httpMethod, HttpClientContext context) throws IOException, InterruptedException { - return client.execute(httpMethod, context); - } - } diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java index e07d39091..20cb051a1 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java @@ -36,8 +36,6 @@ import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.client.CloseableHttpClient; import org.apache.tools.ant.filters.StringInputStream; import static org.mockito.Mockito.mock; @@ -92,10 +90,9 @@ public static class BitbucketServerIntegrationClient extends BitbucketServerAPIC private final String payloadRootPath; private final IRequestAudit audit; - private boolean rateLimitNextRequest; // TODO: Would be nice to have a better way to mock non-200 responses. private BitbucketServerIntegrationClient(String payloadRootPath, String baseURL, String owner, String repositoryName) { - super(baseURL, owner, repositoryName, (BitbucketAuthenticator) null, false); + super(baseURL, owner, repositoryName, mock(BitbucketAuthenticator.class), false); if (payloadRootPath == null) { this.payloadRootPath = PAYLOAD_RESOURCE_ROOTPATH; @@ -107,16 +104,8 @@ private BitbucketServerIntegrationClient(String payloadRootPath, String baseURL, this.audit = mock(IRequestAudit.class); } - public void rateLimitNextRequest() { - rateLimitNextRequest = true; - } - @Override - protected CloseableHttpResponse executeMethodNoRetry(CloseableHttpClient client, HttpRequestBase httpMethod, HttpClientContext context) throws IOException { - if (rateLimitNextRequest) { - rateLimitNextRequest = false; - return createRateLimitResponse(); - } + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { String path = httpMethod.getURI().toString(); audit.request(httpMethod); @@ -160,7 +149,7 @@ private BitbucketClouldIntegrationClient(String payloadRootPath, String owner, S } @Override - protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws InterruptedException, IOException { + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { String path = httpMethod.getURI().toString(); audit.request(httpMethod); diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java index b73fe3853..a98d02e44 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java @@ -7,19 +7,17 @@ import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory; import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory.BitbucketServerIntegrationClient; import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory.IRequestAudit; -import com.damnhandy.uri.template.UriTemplate; -import com.damnhandy.uri.template.impl.Operator; import io.jenkins.cli.shaded.org.apache.commons.lang.RandomStringUtils; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.logging.Level; import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -29,15 +27,13 @@ import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; -import static com.cloudbees.jenkins.plugins.bitbucket.server.client.BitbucketServerAPIClient.API_BROWSE_PATH; import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; @@ -79,36 +75,31 @@ private HttpRequestBase extractRequest(IRequestAudit clientAudit) { @Test @WithoutJenkins - public void repoBrowsePathFolder() { - String expand = UriTemplate - .fromTemplate(API_BROWSE_PATH) - .set("owner", "test") - .set("repo", "test") - .set("path", "folder/Jenkinsfile".split(Operator.PATH.getSeparator())) - .set("at", "fix/test") - .expand(); - Assert.assertEquals("/rest/api/1.0/projects/test/repos/test/browse/folder/Jenkinsfile?at=fix%2Ftest", expand); + public void verify_checkPathExists_given_a_path() throws Exception { + BitbucketApi client = BitbucketIntegrationClientFactory.getApiMockClient("https://acme.bitbucket.org"); + assertThat(client.checkPathExists("feature/pipeline", "folder/Jenkinsfile")).isTrue(); + + IRequestAudit clientAudit = ((IRequestAudit) client).getAudit(); + HttpRequestBase request = extractRequest(clientAudit); + assertThat(request).isNotNull() + .isInstanceOfSatisfying(HttpHead.class, head -> { + assertThat(head.getURI()) + .hasPath("/rest/api/1.0/projects/amuniz/repos/test-repos/browse/folder/Jenkinsfile") + .hasQuery("at=feature/pipeline"); + }); } @Test @WithoutJenkins - public void repoBrowsePathFile() { - String expand = UriTemplate - .fromTemplate(API_BROWSE_PATH) - .set("owner", "test") - .set("repo", "test") - .set("path", "Jenkinsfile".split(Operator.PATH.getSeparator())) - .expand(); - Assert.assertEquals("/rest/api/1.0/projects/test/repos/test/browse/Jenkinsfile", expand); - } + public void verify_checkPathExists_given_file() throws Exception { + BitbucketApi client = BitbucketIntegrationClientFactory.getApiMockClient("https://acme.bitbucket.org"); + assertThat(client.checkPathExists("feature/pipeline", "Jenkinsfile")).isTrue(); - @Test - public void retryWhenRateLimited() throws Exception { - logger.capture(50); - BitbucketApi client = BitbucketIntegrationClientFactory.getClient("localhost", "amuniz", "test-repos"); - ((BitbucketServerIntegrationClient)client).rateLimitNextRequest(); - assertThat(client.getRepository().getProject().getKey(), equalTo("AMUNIZ")); - assertThat(logger.getMessages(), hasItem(containsString("Bitbucket server API rate limit reached"))); + IRequestAudit clientAudit = ((IRequestAudit) client).getAudit(); + HttpRequestBase request = extractRequest(clientAudit); + assertThat(request).isNotNull() + .isInstanceOfSatisfying(HttpHead.class, head -> + assertThat(head.getURI()).hasPath("/rest/api/1.0/projects/amuniz/repos/test-repos/browse/Jenkinsfile")); } @Test @@ -130,10 +121,10 @@ public void sortRepositoriesByName() throws Exception { @Test public void disableCookieManager() throws Exception { - try(MockedStatic staticHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { - HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class); - CloseableHttpClient httpClient = mock(CloseableHttpClient.class); + try (MockedStatic staticHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { + HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class, RETURNS_SELF); staticHttpClientBuilder.when(HttpClientBuilder::create).thenReturn(httpClientBuilder); + CloseableHttpClient httpClient = mock(CloseableHttpClient.class); when(httpClientBuilder.build()).thenReturn(httpClient); BitbucketApi client = BitbucketIntegrationClientFactory.getClient("localhost", "amuniz", "test-repos"); client.getRepositories(); diff --git a/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json new file mode 100644 index 000000000..aca597f12 --- /dev/null +++ b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json @@ -0,0 +1,3 @@ +node() { + echo "ciao" +} \ No newline at end of file diff --git a/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json new file mode 100644 index 000000000..aca597f12 --- /dev/null +++ b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json @@ -0,0 +1,3 @@ +node() { + echo "ciao" +} \ No newline at end of file