Skip to content

Commit

Permalink
feat: add support for retryable download of Node.js distribution (#202)
Browse files Browse the repository at this point in the history
* add retry mechanism to downloads (#200)

* Fixed line endings.

* Fixed Gradle Wrapper mode.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed website code tabs.

* Fixed sitemap.

* Fixed test coverage reporting.

* Fixed website issues.

* Before merging branch 4.0-jdk11.

* Added website publication.

* Updated sitemap.

* Fixed invalid SpringBoot version number in examples

* Initialized release 5.0.1.

* Fixed #153 (#154) (#155)

* Fixed #152 (#156)

* Added contributor.

* Merged fix for #149 (#157)

* Upgraded to Gradle 6.8.3.

* Updated website.

* Initialized release 5.1.0.

* Fixed #148 (#159)

* Merged changes from branch 5.1-jdk8.

* Initialized release 5.2.0.

* Fixed #161 (#163)

* Merged changes from branch 5.2-jdk8.

* Merged changes from branch master.

* Updated documentation.

* Initialized release 5.3.0.

* Fixed #165 (#166)

* Added FAQ section.

* Fixed page title

* chore(release): release 7.0.0

* doc: fixed website publication [skip ci]

* docs: fix web page metadata

* docs: fix web page metadata

* docs: add links to released artifacts, fix descriptions for examples [skip ci]

* docs: fix website publication [skip ci]

* docs: add missing component import [skip ci]

* add retry mechanism to downloads

---------

Co-authored-by: Vincent Bories-Azeau <5869062+vboriesazeau@users.noreply.github.com>
Co-authored-by: Vincent BA <5869062+v1nc3n4@users.noreply.github.com>
Co-authored-by: Jean Detoeuf <Jean.Detoeuf1@ibm.com>

* feat: add support for retryable download of Node.js distribution

Fixes #201

---------

Co-authored-by: Jean Detoeuf <jean.detoeuf@instana.com>
Co-authored-by: Vincent Bories-Azeau <5869062+vboriesazeau@users.noreply.github.com>
Co-authored-by: Jean Detoeuf <Jean.Detoeuf1@ibm.com>
  • Loading branch information
4 people authored Aug 18, 2023
1 parent f1c4740 commit 940f5d0
Show file tree
Hide file tree
Showing 44 changed files with 912 additions and 322 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ With their feedback, plugin improvement is possible. Special thanks to:
@[rolaca11](https://github.com/rolaca11),
@[stephanebastian](https://github.com/stephanebastian),
@[TapaiBalazs](https://github.com/TapaiBalazs),
@[thebignet](https://github.com/thebignet)
@[tngwoerleij](https://github.com/tngwoerleij),
@[trohr](https://github.com/trohr),
@[xehonk](https://github.com/xehonk)
Expand Down
1 change: 1 addition & 0 deletions plugin/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ configurations["intTestRuntimeOnly"]

dependencies {
implementation(gradleApi())
implementation("io.github.resilience4j:resilience4j-retry:2.1.0")
implementation("org.apache.httpcomponents.client5:httpclient5:5.2.1")
implementation("org.apache.commons:commons-compress:1.23.0")
implementation("org.json:json:20230227")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Set;

import org.gradle.api.GradleException;
import org.gradle.api.Plugin;
Expand Down Expand Up @@ -97,6 +98,11 @@ public class FrontendGradlePlugin implements Plugin<Project> {
*/
public static final String DEFAULT_INSTALL_SCRIPT = "install";

/**
* Maximum number of attempts to download a file.
*/
public static final int DEFAULT_MAX_DOWNLOAD_ATTEMPTS = 1;

/**
* Name of the task that installs a Node.js distribution.
*/
Expand All @@ -112,6 +118,17 @@ public class FrontendGradlePlugin implements Plugin<Project> {
*/
public static final String DEFAULT_NODE_DISTRIBUTION_URL_ROOT = "https://nodejs.org/dist/";

/**
* HTTP statuses that should trigger another download attempt.
*/
public static final Set<Integer> DEFAULT_RETRY_HTTP_STATUSES = Set.of(408, 429, 500, 502, 503, 504);

public static final int DEFAULT_RETRY_INITIAL_INTERVAL_MS = 1000;

public static final double DEFAULT_RETRY_INTERVAL_MULTIPLIER = 2;

public static final int DEFAULT_RETRY_MAX_INTERVAL_MS = 30000;

public static final String GRADLE_ASSEMBLE_TASK_NAME = LifecycleBasePlugin.ASSEMBLE_TASK_NAME;

public static final String GRADLE_CHECK_TASK_NAME = LifecycleBasePlugin.CHECK_TASK_NAME;
Expand Down Expand Up @@ -188,6 +205,11 @@ public void apply(final Project project) {
frontendExtension.getPackageJsonDirectory().convention(project.getLayout().getProjectDirectory());
frontendExtension.getHttpProxyPort().convention(DEFAULT_HTTP_PROXY_PORT);
frontendExtension.getHttpsProxyPort().convention(DEFAULT_HTTPS_PROXY_PORT);
frontendExtension.getMaxDownloadAttempts().convention(DEFAULT_MAX_DOWNLOAD_ATTEMPTS);
frontendExtension.getRetryHttpStatuses().convention(DEFAULT_RETRY_HTTP_STATUSES);
frontendExtension.getRetryInitialIntervalMs().convention(DEFAULT_RETRY_INITIAL_INTERVAL_MS);
frontendExtension.getRetryIntervalMultiplier().convention(DEFAULT_RETRY_INTERVAL_MULTIPLIER);
frontendExtension.getRetryMaxIntervalMs().convention(DEFAULT_RETRY_MAX_INTERVAL_MS);
frontendExtension
.getCacheDirectory()
.convention(project.getLayout().getProjectDirectory().dir(DEFAULT_CACHE_DIRECTORY_NAME));
Expand Down Expand Up @@ -341,6 +363,11 @@ protected void configureInstallNodeTask(final InstallNodeTask task, final String
.platform(getBeanOrFail(beanRegistryId, PlatformProvider.class).getPlatform())
.build())
.toFile()));
task.getMaxDownloadAttempts().set(extension.getMaxDownloadAttempts());
task.getRetryHttpStatuses().set(extension.getRetryHttpStatuses());
task.getRetryInitialIntervalMs().set(extension.getRetryInitialIntervalMs());
task.getRetryIntervalMultiplier().set(extension.getRetryIntervalMultiplier());
task.getRetryMaxIntervalMs().set(extension.getRetryMaxIntervalMs());
task.setOnlyIf(t -> !extension.getNodeDistributionProvided().get());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,8 @@ public abstract class FrontendException extends Exception {
protected FrontendException(final String message) {
super(message);
}

protected FrontendException(final Throwable throwable) {
super(throwable);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public void execute(final DeployDistributionCommand command)
throws UnsupportedDistributionArchiveException, ArchiverException, IOException {
// Explodes the archive
final Path temporaryDirectoryPath = fileManager.createDirectory(command.getTemporaryDirectoryPath());

logger.info("Exploding distribution into '{}'", temporaryDirectoryPath);
final Path distributionFilePath = command.getDistributionFilePath();
archiverProvider
Expand All @@ -66,8 +67,5 @@ public void execute(final DeployDistributionCommand command)
distributionRootDirectoryPath = temporaryDirectoryPath;
}
fileManager.moveFileTree(distributionRootDirectoryPath, installDirectoryPath);

logger.info("Removing explode directory '{}'", temporaryDirectoryPath);
fileManager.deleteIfExists(temporaryDirectoryPath);
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package org.siouan.frontendgradleplugin.domain.installer;

import java.io.IOException;
import java.net.URL;
import java.net.Proxy;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;

import io.github.resilience4j.core.IntervalFunction;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryConfig;
import io.github.resilience4j.retry.event.RetryEvent;
import lombok.RequiredArgsConstructor;
import org.siouan.frontendgradleplugin.domain.ChannelProvider;
import org.siouan.frontendgradleplugin.domain.FileManager;
Expand All @@ -19,6 +23,8 @@
@RequiredArgsConstructor
public class DownloadResource {

private static final String RETRY_ID = DownloadResource.class.getSimpleName();

private final FileManager fileManager;

private final ChannelProvider channelProvider;
Expand All @@ -32,45 +38,92 @@ public class DownloadResource {
* the caller to ensure the temporary file is writable, and the directory receiving the destination file exists and
* is writable.
*
* @param downloadResourceCommand Command providing download parameters.
* @param command Command providing download parameters.
* @throws IOException If the resource transfer failed, or the resource could not be written in the temporary file,
* or moved to the destination file. In this case, any temporary file created is removed.
* @throws ResourceDownloadException If the distribution download failed.
* @throws ResourceDownloadException If the resource download failed .
*/
public void execute(final DownloadResourceCommand downloadResourceCommand)
throws IOException, ResourceDownloadException {
final URL resourceUrl = downloadResourceCommand.getResourceUrl();
final ProxySettings proxySettings = downloadResourceCommand.getProxySettings();
public void execute(final DownloadResourceCommand command) throws IOException, ResourceDownloadException {
// Configure retry with exponential backoff.
final RetrySettings retrySettings = command.getRetrySettings();
final RetryConfig retryConfig = RetryConfig
.<Void>custom()
.maxAttempts(retrySettings.getMaxDownloadAttempts())
.intervalFunction(IntervalFunction.ofExponentialBackoff(retrySettings.getRetryInitialIntervalMs(),
retrySettings.getRetryIntervalMultiplier(), retrySettings.getRetryMaxIntervalMs()))
.retryExceptions(RetryableResourceDownloadException.class)
.build();
final Retry retry = Retry.of(RETRY_ID, retryConfig);
retry
.getEventPublisher()
.onRetry(event -> logAttemptFailure(retrySettings, event))
.onError(event -> logAttemptFailure(retrySettings, event));

// Download resource.
try {
Retry.decorateCheckedRunnable(retry, () -> attemptDownload(command)).run();
} catch (final Throwable e) {
// At this point, a single type of exception is likely to be catched. But in case of a bug, type casting may
// fail and no information will be logged about the real exception catched. That's why logging the exception
// is a precondition to ease troubleshooting with end-users.
logger.debug("All download attempts failed", e);
throw (ResourceDownloadException) e;
}

fileManager.move(command.getTemporaryFilePath(), command.getDestinationFilePath(),
StandardCopyOption.REPLACE_EXISTING);
}

if (proxySettings == null) {
logger.info("Downloading resource at '{}' (proxy: DIRECT)", downloadResourceCommand.getResourceUrl());
private void attemptDownload(final DownloadResourceCommand command) throws ResourceDownloadException {
final ProxySettings proxySettings = command.getProxySettings();
if (proxySettings.getProxyType() == Proxy.Type.DIRECT) {
logger.info("Downloading resource at '{}' (proxy: DIRECT)", command.getResourceUrl());
} else {
logger.info("Downloading resource at '{}' (proxy: {}/{}:{})", downloadResourceCommand.getResourceUrl(),
logger.info("Downloading resource at '{}' (proxy: {}/{}:{})", command.getResourceUrl(),
proxySettings.getProxyType(), proxySettings.getProxyHost(), proxySettings.getProxyPort());
}

try (final HttpResponse response = httpClientProvider
.getInstance()
.sendGetRequest(resourceUrl, downloadResourceCommand.getServerCredentials(), proxySettings);
final HttpClient httpClient = httpClientProvider.getInstance();
try (final HttpResponse response = httpClient.sendGetRequest(command.getResourceUrl(),
command.getServerCredentials(), command.getProxySettings());
final ReadableByteChannel resourceInputChannel = channelProvider.getReadableByteChannel(
response.getInputStream());
final FileChannel resourceOutputChannel = channelProvider.getWritableFileChannelForNewFile(
downloadResourceCommand.getTemporaryFilePath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
command.getTemporaryFilePath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING)) {
logger.debug("---> {}/{} {} {}", response.getProtocol(), response.getVersion(), response.getStatusCode(),
final int statusCode = response.getStatusCode();
logger.debug("---> {}/{} {} {}", response.getProtocol(), response.getVersion(), statusCode,
response.getReasonPhrase());
if (response.getStatusCode() != 200) {
throw new ResourceDownloadException(
if (statusCode == 200) {
resourceOutputChannel.transferFrom(resourceInputChannel, 0, Long.MAX_VALUE);
} else {
final String errorMessage =
"Unexpected HTTP response: " + response.getProtocol() + '/' + response.getVersion() + ' '
+ response.getStatusCode() + ' ' + response.getReasonPhrase());
+ statusCode + ' ' + response.getReasonPhrase();
if (command.getRetrySettings().getRetryHttpStatuses().contains(statusCode)) {
throw new RetryableResourceDownloadException(errorMessage);
} else {
// Download failed because the server responded with a non-retryable HTTP status code.
throw new ResourceDownloadException(errorMessage);
}
}
} catch (final IOException e) {
try {
fileManager.deleteIfExists(command.getTemporaryFilePath());
} catch (IOException ex) {
// Ignore this second exception as it would hide the first exception.
logger.warn("Unexpected error while deleting file after error: {}", command.getTemporaryFilePath());
}
resourceOutputChannel.transferFrom(resourceInputChannel, 0, Long.MAX_VALUE);
} catch (final IOException | ResourceDownloadException e) {
fileManager.deleteIfExists(downloadResourceCommand.getTemporaryFilePath());
throw e;
throw new RetryableResourceDownloadException(e);
}
}

fileManager.move(downloadResourceCommand.getTemporaryFilePath(),
downloadResourceCommand.getDestinationFilePath(), StandardCopyOption.REPLACE_EXISTING);
private void logAttemptFailure(final RetrySettings retrySettings, final RetryEvent retryEvent) {
final int maxDownloadAttempts = retrySettings.getMaxDownloadAttempts();
// Log attempt exception message just in case retry is enabled.
if (maxDownloadAttempts > 1) {
logger.info("Attempt {} of {}: {}", retryEvent.getNumberOfRetryAttempts(), maxDownloadAttempts,
retryEvent.getLastThrowable().getMessage());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ public class DownloadResourceCommand {
@EqualsAndHashCode.Include
private final ProxySettings proxySettings;

/**
* Settings to retry a file download.
*
* @since 7.1.0
*/
@EqualsAndHashCode.Include
private final RetrySettings retrySettings;

/**
* Path to a temporary file.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import org.siouan.frontendgradleplugin.domain.UnsupportedPlatformException;

/**
* Downloads and optionally validates a distribution file.
* Downloads and validates a distribution file.
*
* @since 2.0.0
*/
Expand Down Expand Up @@ -74,14 +74,20 @@ public Path execute(final GetDistributionCommand command)
.resourceUrl(distributionUrl)
.serverCredentials(command.getDistributionServerCredentials())
.proxySettings(command.getProxySettings())
.retrySettings(command.getRetrySettings())
.temporaryFilePath(temporaryFilePath)
.destinationFilePath(distributionFilePath)
.build());

final ValidateNodeDistributionCommand validateNodeDistributionCommand = new ValidateNodeDistributionCommand(
command.getTemporaryDirectoryPath(), distributionUrl, command.getDistributionServerCredentials(),
command.getProxySettings(), distributionFilePath);
validateNodeDistribution.execute(validateNodeDistributionCommand);
validateNodeDistribution.execute(ValidateNodeDistributionCommand
.builder()
.temporaryDirectoryPath(command.getTemporaryDirectoryPath())
.distributionUrl(distributionUrl)
.distributionServerCredentials(command.getDistributionServerCredentials())
.proxySettings(command.getProxySettings())
.retrySettings(command.getRetrySettings())
.distributionFilePath(distributionFilePath)
.build());

return distributionFilePath;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ public class GetDistributionCommand {
@EqualsAndHashCode.Include
private final ProxySettings proxySettings;

/**
* Settings to retry a file download.
*
* @since 7.1.0
*/
@EqualsAndHashCode.Include
private final RetrySettings retrySettings;

/**
* Path to a temporary directory.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,18 @@ public void execute(final InstallNodeDistributionCommand command)

final GetDistributionCommand getDistributionCommand = new GetDistributionCommand(command.platform(),
command.version(), command.distributionUrlRoot(), command.distributionUrlPathPattern(),
command.distributionServerCredentials(), command.proxySettings(), command.temporaryDirectoryPath());
command.distributionServerCredentials(), command.proxySettings(), command.retrySettings(),
command.temporaryDirectoryPath());
final Path distributionFilePath = getDistribution.execute(getDistributionCommand);

// Deploys the distribution
deployDistribution.execute(new DeployDistributionCommand(command.platform(),
command.temporaryDirectoryPath().resolve(EXTRACT_DIRECTORY_NAME), command.installDirectoryPath(),
distributionFilePath));
final Path temporaryExplodeDirectoryPath = command.temporaryDirectoryPath().resolve(EXTRACT_DIRECTORY_NAME);
// In case deployment fails, the plugin should leave the file system in a consistent state and delete
// temporary resources created.
logger.debug("Removing explode directory '{}'", temporaryExplodeDirectoryPath);
fileManager.deleteFileTree(temporaryExplodeDirectoryPath, true);
deployDistribution.execute(new DeployDistributionCommand(command.platform(), temporaryExplodeDirectoryPath,
command.installDirectoryPath(), distributionFilePath));

logger.info("Removing distribution file '{}'", distributionFilePath);
fileManager.delete(distributionFilePath);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@
* @param distributionUrlPathPattern Trailing path pattern to build the exact URL to download the distribution.
* @param distributionServerCredentials Credentials to authenticate on the distribution server before download.
* @param proxySettings Proxy settings used for downloads.
* @param retrySettings Settings to retry a file download.
* @param temporaryDirectoryPath Path to a temporary directory.
* @param installDirectoryPath Path to a directory where the distribution shall be installed.
* @since 1.1.2
*/
@Builder
public record InstallNodeDistributionCommand(Platform platform, String version, String distributionUrlRoot,
String distributionUrlPathPattern, Credentials distributionServerCredentials, ProxySettings proxySettings,
Path temporaryDirectoryPath, Path installDirectoryPath) {}
RetrySettings retrySettings, Path temporaryDirectoryPath, Path installDirectoryPath) {}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class ProxySettings {

/**
* Settings that request a direct connection.
*/
public static final ProxySettings NONE = ProxySettings.builder().proxyType(Proxy.Type.DIRECT).build();

/**
* Proxy host.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public ProxySettings execute(final ResolveProxySettingsByUrlCommand command) {
.nonProxyHosts(systemSettingsProvider.getNonProxyHosts())
.hostNameOrIpAddress(resourceUrl.getHost())
.build())) {
return null;
return ProxySettings.NONE;
} else {
final SelectProxySettingsCommand.SelectProxySettingsCommandBuilder selectProxySettingsCommandBuilder = SelectProxySettingsCommand.builder();
if (resourceProtocol.equals(HTTPS_PROTOCOL)) {
Expand All @@ -55,7 +55,7 @@ public ProxySettings execute(final ResolveProxySettingsByUrlCommand command) {
return selectProxySettings.execute(selectProxySettingsCommandBuilder.build());
}
} else if (resourceProtocol.equals(FILE_PROTOCOL)) {
return null;
return ProxySettings.NONE;
} else {
throw new IllegalArgumentException("Unsupported protocol: " + resourceUrl.getProtocol());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import org.siouan.frontendgradleplugin.domain.FrontendException;

/**
* Exception thrown when a resource download does not complete with a HTTP 200 response.
* Exception thrown when a resource download definitely failed despite eventual multiple attempts.
*
* @since 4.0.1
*/
Expand All @@ -12,4 +12,8 @@ public class ResourceDownloadException extends FrontendException {
public ResourceDownloadException(final String message) {
super(message);
}

public ResourceDownloadException(final Throwable e) {
super(e);
}
}
Loading

0 comments on commit 940f5d0

Please sign in to comment.