-
Notifications
You must be signed in to change notification settings - Fork 188
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
103 changed files
with
1,172 additions
and
236 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
src/main/java/org/prebid/server/auction/model/TimeoutContext.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 0 additions & 12 deletions
12
src/main/java/org/prebid/server/execution/RemoteFileProcessor.java
This file was deleted.
Oops, something went wrong.
8 changes: 8 additions & 0 deletions
8
src/main/java/org/prebid/server/execution/file/FileProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package org.prebid.server.execution.file; | ||
|
|
||
| import io.vertx.core.Future; | ||
|
|
||
| public interface FileProcessor { | ||
|
|
||
| Future<?> setDataPath(String dataFilePath); | ||
| } |
106 changes: 106 additions & 0 deletions
106
src/main/java/org/prebid/server/execution/file/FileUtil.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package org.prebid.server.execution.file; | ||
|
|
||
| import io.vertx.core.Vertx; | ||
| import io.vertx.core.file.FileProps; | ||
| import io.vertx.core.file.FileSystem; | ||
| import io.vertx.core.file.FileSystemException; | ||
| import io.vertx.core.http.HttpClientOptions; | ||
| import org.apache.commons.lang3.ObjectUtils; | ||
| import org.prebid.server.exception.PreBidException; | ||
| import org.prebid.server.execution.file.syncer.FileSyncer; | ||
| import org.prebid.server.execution.file.syncer.LocalFileSyncer; | ||
| import org.prebid.server.execution.file.syncer.RemoteFileSyncerV2; | ||
| import org.prebid.server.execution.retry.ExponentialBackoffRetryPolicy; | ||
| import org.prebid.server.execution.retry.FixedIntervalRetryPolicy; | ||
| import org.prebid.server.execution.retry.RetryPolicy; | ||
| import org.prebid.server.spring.config.model.ExponentialBackoffProperties; | ||
| import org.prebid.server.spring.config.model.FileSyncerProperties; | ||
| import org.prebid.server.spring.config.model.HttpClientProperties; | ||
|
|
||
| import java.nio.file.Files; | ||
| import java.nio.file.InvalidPathException; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
|
|
||
| public class FileUtil { | ||
|
|
||
| private FileUtil() { | ||
| } | ||
|
|
||
| public static void createAndCheckWritePermissionsFor(FileSystem fileSystem, String filePath) { | ||
| try { | ||
| final Path dirPath = Paths.get(filePath).getParent(); | ||
| final String dirPathString = dirPath.toString(); | ||
| final FileProps props = fileSystem.existsBlocking(dirPathString) | ||
| ? fileSystem.propsBlocking(dirPathString) | ||
| : null; | ||
|
|
||
| if (props == null || !props.isDirectory()) { | ||
| fileSystem.mkdirsBlocking(dirPathString); | ||
| } else if (!Files.isWritable(dirPath)) { | ||
| throw new PreBidException("No write permissions for directory: " + dirPath); | ||
| } | ||
| } catch (FileSystemException | InvalidPathException e) { | ||
| throw new PreBidException("Cannot create directory for file: " + filePath, e); | ||
| } | ||
| } | ||
|
|
||
| public static FileSyncer fileSyncerFor(FileProcessor fileProcessor, | ||
| FileSyncerProperties properties, | ||
| Vertx vertx) { | ||
|
|
||
| return switch (properties.getType()) { | ||
| case LOCAL -> new LocalFileSyncer( | ||
| fileProcessor, | ||
| properties.getSaveFilepath(), | ||
| properties.getUpdateIntervalMs(), | ||
| toRetryPolicy(properties), | ||
| vertx); | ||
| case REMOTE -> remoteFileSyncer(fileProcessor, properties, vertx); | ||
| }; | ||
| } | ||
|
|
||
| private static RemoteFileSyncerV2 remoteFileSyncer(FileProcessor fileProcessor, | ||
| FileSyncerProperties properties, | ||
| Vertx vertx) { | ||
|
|
||
| final HttpClientProperties httpClientProperties = properties.getHttpClient(); | ||
| final HttpClientOptions httpClientOptions = new HttpClientOptions() | ||
| .setConnectTimeout(httpClientProperties.getConnectTimeoutMs()) | ||
| .setMaxRedirects(httpClientProperties.getMaxRedirects()); | ||
|
|
||
| return new RemoteFileSyncerV2( | ||
| fileProcessor, | ||
| properties.getDownloadUrl(), | ||
| properties.getSaveFilepath(), | ||
| properties.getTmpFilepath(), | ||
| vertx.createHttpClient(httpClientOptions), | ||
| properties.getTimeoutMs(), | ||
| properties.isCheckSize(), | ||
| properties.getUpdateIntervalMs(), | ||
| toRetryPolicy(properties), | ||
| vertx); | ||
| } | ||
|
|
||
| // TODO: remove after transition period | ||
| private static RetryPolicy toRetryPolicy(FileSyncerProperties properties) { | ||
| final Long retryIntervalMs = properties.getRetryIntervalMs(); | ||
| final Integer retryCount = properties.getRetryCount(); | ||
| final boolean fixedRetryPolicyDefined = ObjectUtils.anyNotNull(retryIntervalMs, retryCount); | ||
| final boolean fixedRetryPolicyValid = ObjectUtils.allNotNull(retryIntervalMs, retryCount) | ||
| || !fixedRetryPolicyDefined; | ||
|
|
||
| if (!fixedRetryPolicyValid) { | ||
| throw new IllegalArgumentException("fixed interval retry policy is invalid"); | ||
| } | ||
|
|
||
| final ExponentialBackoffProperties exponentialBackoffProperties = properties.getRetry(); | ||
| return fixedRetryPolicyDefined | ||
| ? FixedIntervalRetryPolicy.limited(retryIntervalMs, retryCount) | ||
| : ExponentialBackoffRetryPolicy.of( | ||
| exponentialBackoffProperties.getDelayMillis(), | ||
| exponentialBackoffProperties.getMaxDelayMillis(), | ||
| exponentialBackoffProperties.getFactor(), | ||
| exponentialBackoffProperties.getJitter()); | ||
| } | ||
| } |
47 changes: 47 additions & 0 deletions
47
src/main/java/org/prebid/server/execution/file/supplier/LocalFileSupplier.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package org.prebid.server.execution.file.supplier; | ||
|
|
||
| import io.vertx.core.Future; | ||
| import io.vertx.core.file.FileProps; | ||
| import io.vertx.core.file.FileSystem; | ||
|
|
||
| import java.util.Objects; | ||
| import java.util.concurrent.atomic.AtomicLong; | ||
| import java.util.function.Supplier; | ||
|
|
||
| public class LocalFileSupplier implements Supplier<Future<String>> { | ||
|
|
||
| private final String filePath; | ||
| private final FileSystem fileSystem; | ||
| private final AtomicLong lastSupplyTime; | ||
|
|
||
| public LocalFileSupplier(String filePath, FileSystem fileSystem) { | ||
| this.filePath = Objects.requireNonNull(filePath); | ||
| this.fileSystem = Objects.requireNonNull(fileSystem); | ||
| lastSupplyTime = new AtomicLong(Long.MIN_VALUE); | ||
| } | ||
|
|
||
| @Override | ||
| public Future<String> get() { | ||
| return fileSystem.exists(filePath) | ||
| .compose(exists -> exists | ||
| ? fileSystem.props(filePath) | ||
| : Future.failedFuture("File %s not found.".formatted(filePath))) | ||
| .map(this::getFileIfModified); | ||
| } | ||
|
|
||
| private String getFileIfModified(FileProps fileProps) { | ||
| final long lastModifiedTime = lasModifiedTime(fileProps); | ||
| final long lastSupplyTime = this.lastSupplyTime.get(); | ||
|
|
||
| if (lastSupplyTime < lastModifiedTime) { | ||
| this.lastSupplyTime.compareAndSet(lastSupplyTime, lastModifiedTime); | ||
| return filePath; | ||
| } | ||
|
|
||
| return null; | ||
| } | ||
|
|
||
| private static long lasModifiedTime(FileProps fileProps) { | ||
| return Math.max(fileProps.creationTime(), fileProps.lastModifiedTime()); | ||
| } | ||
| } |
Oops, something went wrong.