-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add interface changes for async repository downloads
Signed-off-by: Kunal Kotwani <kkotwani@amazon.com>
- Loading branch information
1 parent
8e95a82
commit b505545
Showing
11 changed files
with
394 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
...src/main/java/org/opensearch/common/blobstore/stream/listener/FileCompletionListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.blobstore.stream.listener; | ||
|
|
||
| import org.opensearch.core.action.ActionListener; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.util.List; | ||
| import java.util.Set; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
|
|
||
| public class FileCompletionListener implements ActionListener<String> { | ||
| private final int numStreams; | ||
| private final String segmentFileName; | ||
| private final Path segmentDirectory; | ||
| private final List<String> toDownloadPartFileNames; | ||
| private final AtomicInteger downloadedPartFiles; | ||
| private final AtomicBoolean anyStreamFailed; | ||
| private final ActionListener<String> segmentCompletionListener; | ||
|
|
||
| public FileCompletionListener( | ||
| int numStreams, | ||
| String segmentFileName, | ||
| Path segmentDirectory, | ||
| List<String> toDownloadPartFileNames, | ||
| AtomicBoolean anyStreamFailed, | ||
| ActionListener<String> segmentCompletionListener | ||
| ) { | ||
| this.downloadedPartFiles = new AtomicInteger(); | ||
| this.numStreams = numStreams; | ||
| this.segmentFileName = segmentFileName; | ||
| this.segmentDirectory = segmentDirectory; | ||
| this.anyStreamFailed = anyStreamFailed; | ||
| this.toDownloadPartFileNames = toDownloadPartFileNames; | ||
| this.segmentCompletionListener = segmentCompletionListener; | ||
| } | ||
|
|
||
| @Override | ||
| public void onResponse(String streamFileName) { | ||
| if (!anyStreamFailed.get() && downloadedPartFiles.incrementAndGet() == numStreams) { | ||
| createCompleteSegmentFile(); | ||
| performChecksum(); | ||
| segmentCompletionListener.onResponse(segmentFileName); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void onFailure(Exception e) { | ||
| try (Stream<Path> segmentDirectoryStream = Files.list(segmentDirectory)) { | ||
| Set<String> tempFilesInDirectory = segmentDirectoryStream.filter(path -> !Files.isDirectory(path)) | ||
| .map(path -> path.getFileName().toString()) | ||
| .collect(Collectors.toSet()); | ||
|
|
||
| if (tempFilesInDirectory.contains(segmentFileName)) { | ||
| Files.delete(segmentDirectory.resolve(segmentFileName)); | ||
| } | ||
|
|
||
| tempFilesInDirectory.retainAll(toDownloadPartFileNames); | ||
| for (String tempFile : tempFilesInDirectory) { | ||
| Files.delete(segmentDirectory.resolve(tempFile)); | ||
| } | ||
|
|
||
| } catch (IOException ex) { | ||
| // Die silently? | ||
| } | ||
|
|
||
| if (!anyStreamFailed.get()) { | ||
| segmentCompletionListener.onFailure(e); | ||
| anyStreamFailed.compareAndSet(false, true); | ||
| } | ||
| } | ||
|
|
||
| private void performChecksum() { | ||
| // TODO: Add checksum logic | ||
| } | ||
|
|
||
| private void createCompleteSegmentFile() { | ||
| try { | ||
| Path segmentFilePath = segmentDirectory.resolve(segmentFileName); | ||
| try (OutputStream segmentFile = Files.newOutputStream(segmentFilePath)) { | ||
| for (String partFileName : toDownloadPartFileNames) { | ||
| Path partFilePath = segmentDirectory.resolve(partFileName); | ||
| try (InputStream partFile = Files.newInputStream(partFilePath)) { | ||
| partFile.transferTo(segmentFile); | ||
| } | ||
| Files.delete(partFilePath); | ||
| } | ||
| } | ||
| } catch (IOException e) { | ||
| onFailure(e); | ||
| } | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
...c/main/java/org/opensearch/common/blobstore/stream/listener/StreamCompletionListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.blobstore.stream.listener; | ||
|
|
||
| import org.opensearch.core.action.ActionListener; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| public class StreamCompletionListener implements ActionListener<InputStream> { | ||
| private final String partFileName; | ||
| private final Path segmentDirectory; | ||
| private final AtomicBoolean anyStreamFailed; | ||
| private final ActionListener<String> fileCompletionListener; | ||
|
|
||
| public StreamCompletionListener( | ||
| String partFileName, | ||
| Path segmentDirectory, | ||
| AtomicBoolean anyStreamFailed, | ||
| ActionListener<String> fileCompletionListener | ||
| ) { | ||
| this.partFileName = partFileName; | ||
| this.segmentDirectory = segmentDirectory; | ||
| this.anyStreamFailed = anyStreamFailed; | ||
| this.fileCompletionListener = fileCompletionListener; | ||
| } | ||
|
|
||
| @Override | ||
| public void onResponse(InputStream inputStream) { | ||
| try (inputStream) { | ||
| // Do not write new segments if any stream for this file has already failed | ||
| if (!anyStreamFailed.get()) { | ||
| Path partFilePath = segmentDirectory.resolve(partFileName); | ||
| try (OutputStream outputStream = Files.newOutputStream(partFilePath)) { | ||
| inputStream.transferTo(outputStream); | ||
| } | ||
| fileCompletionListener.onResponse(partFileName); | ||
| } | ||
| } catch (IOException e) { | ||
| onFailure(e); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void onFailure(Exception e) { | ||
| fileCompletionListener.onFailure(e); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.