diff --git a/AZURE.md b/AZURE.md new file mode 100644 index 00000000..de5b07ef --- /dev/null +++ b/AZURE.md @@ -0,0 +1,44 @@ +# Azure Documentation + +## Server Setup + +### Storage Container Setup +Assuming you have a storage account already created inside a resource group +in Azure, create a container called `data`. + +Upload a sentinel object into the container. This can be a small text file that +score will use to validate connectivity to the object storage at request time. + +Create access policies on the storage container. One for read operations and one +for write operations. +Example: +![Access Policies](./docs/img/azure_policies.png) + +Configure the score server either in the config file or env vars: + +```yml +object: + sentinel: # name of the sentinel object you upoaded + +azure: + endpointProtocol: https + accountName: + accountKey: + +bucket: + name.object: data # container name + policy.upload: # name of the policy for the write/add/modify operations + policy.download: # name of policy for the read/list operations + +download: + partsize: 250000000 # safe default +``` + +Start the server with the following run profiles enabled: `prod,secure,azure` + +## Client Setup +Follow the regular client setup of access token and storage and metadata urls. +However when running the client use azure pofile option: +```bash +$ bin/score-client --profile azure ... +``` \ No newline at end of file diff --git a/docs/img/azure_policies.png b/docs/img/azure_policies.png new file mode 100644 index 00000000..a06ff45e Binary files /dev/null and b/docs/img/azure_policies.png differ diff --git a/pom.xml b/pom.xml index b07018ae..ca0e7485 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 pom ${project.artifactId} diff --git a/score-client/pom.xml b/score-client/pom.xml index dae999b6..1ac34d59 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 ../pom.xml diff --git a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java index b2ebdc87..53c78f16 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java @@ -41,34 +41,24 @@ public class UploadStateStore extends TransferState { * /path/to/./uploadid upload state file */ - /* - * We are using the directory that the upload file is currently located in. Requires read/write access to directory - * Otherwise, we need to have user specify a working directory for upload - which is a bit too counter-intuitive. - */ @SneakyThrows - public static File getContainingDir(@NonNull final File uploadFile) { - val parentPath = Optional.ofNullable(uploadFile.getParent()); - if (parentPath.isPresent()) { - return new File(parentPath.get()); - } - return null; + public static String getContainingDir(@NonNull final File uploadFile) { + return uploadFile.getParent(); } /** * Write upload-id of current upload into state directory (hidden directory next to file being uploaded) - * @param uploadFile - File we want to upload. Used to determine path to create temporary upload id directory + * @param uploadStateDir - Path to create temporary upload id directory * @param spec - * @param force */ - public static void create(@NonNull File uploadFile, @NonNull ObjectSpecification spec, boolean force) { - val filePath = getContainingDir(uploadFile); - + public static void create(@NonNull String uploadStateDir, @NonNull ObjectSpecification spec) throws NotRetryableException { try { - val uploadStateDir = getObjectStateDir(filePath, spec.getObjectId()); + val objectStatePath = getObjectStatePath(uploadStateDir, spec.getObjectId()); + val objectStateDir = new File(objectStatePath); - removeDir(uploadStateDir, true); + removeDir(objectStateDir, true); - val uploadIdFile = new File(uploadStateDir, getStateName()); + val uploadIdFile = new File(objectStateDir, getStateName()); try (PrintWriter out = new PrintWriter(uploadIdFile, StandardCharsets.UTF_8.name())) { out.println(spec.getUploadId()); } @@ -82,10 +72,10 @@ protected static String getStateName() { return "uploadId"; } - public static Optional fetchUploadId(@NonNull File uploadFile, @NonNull String objectId) { - Optional result = Optional.ofNullable(null); - val uploadStateDir = getObjectStateDir(getContainingDir(uploadFile), objectId); - val uploadIdFile = new File(uploadStateDir, getStateName()); + public static Optional fetchUploadId(@NonNull String uploadStateDir, @NonNull String objectId) { + Optional result = Optional.empty(); + val objectStatePath = getObjectStatePath(uploadStateDir, objectId); + val uploadIdFile = new File(objectStatePath, getStateName()); if (uploadIdFile.exists()) { try (val reader = @@ -103,4 +93,13 @@ public static Optional fetchUploadId(@NonNull File uploadFile, @NonNull } return result; } + + public static void close(@NonNull String uploadStateDir, @NonNull String objectId) throws IOException { + val dirToDelete = new File(getObjectStatePath(uploadStateDir, objectId)); + deleteDirectoryIfExist(dirToDelete); + } + + private static String getObjectStatePath(@NonNull String uploadStateDir, @NonNull String objectId) { + return uploadStateDir + "/." + objectId; + } } diff --git a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java index f5e11361..61ac941b 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java @@ -52,6 +52,8 @@ public class S3UploadService implements UploadService { private boolean quiet; @Value("${storage.retryNumber}") private int retryNumber; + @Value("${client.uploadStateDir}") + private String uploadStateDir; /** * Dependencies. @@ -104,16 +106,18 @@ private void startUpload(File file, String objectId, String md5, boolean overwri ObjectSpecification spec = null; try { spec = storageService.initiateUpload(objectId, file.length(), overwrite, md5); + + // Delete if already present + if (overwrite) { + UploadStateStore.create(getUploadStateDir(file), spec); + } } catch (NotRetryableException e) { // A NotRetryable exception during initiateUpload should just end whole process // a bit of a sleazy hack. Should only be thrown when the Metadata service informs us the supplied // object id was never registered/does not exist in Metadata repo - throw new NotResumableException(e); - } - // Delete if already present - if (overwrite) { - UploadStateStore.create(file, spec, false); + // being unable to create file for upload state should also end whole process + throw new NotResumableException(e); } val progress = new Progress(terminal, quiet, spec.getParts().size(), 0); @@ -144,7 +148,7 @@ private UploadProgress checkProgress(File uploadFile, String objectId) { // See if there is already an upload in progress for this object id. Fetch upload id and send if present. If // missing, send null - val uploadId = UploadStateStore.fetchUploadId(uploadFile, objectId); + val uploadId = UploadStateStore.fetchUploadId(getUploadStateDir(uploadFile), objectId); val progress = storageService.getProgress(objectId, uploadFile.length()); // Compare upload id's @@ -231,7 +235,14 @@ public boolean isObjectExist(String objectId) throws IOException { } private void cleanupState(File uploadFile, String objectId) throws IOException { - UploadStateStore.close(UploadStateStore.getContainingDir(uploadFile), objectId); + UploadStateStore.close(getUploadStateDir(uploadFile), objectId); } + private String getUploadStateDir(File uploadFile) { + if (!uploadStateDir.isEmpty()) { + return uploadStateDir; + } else { + return UploadStateStore.getContainingDir(uploadFile); + } + } } diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 45789217..40d955ba 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -29,6 +29,9 @@ client: accessToken: "${accessToken:}" # Alias for backwards compatibility connectTimeoutSeconds: 60 readTimeoutSeconds: 60 + # path of dir with WRITE access for score client upload state files + # if empty uses parent dir of current file to upload + uploadStateDir: "" ssl: custom: false diff --git a/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java b/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java index 07f7fbef..8485cf9d 100644 --- a/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java +++ b/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java @@ -11,8 +11,8 @@ public class UploadStateStoreTests { - File getTestResourceRoot() { - return new File(getClass().getClassLoader().getResource("fixtures/upload/placeholder-upload-file.txt").getFile()); + String getTestResourceRoot() { + return getClass().getClassLoader().getResource("fixtures/upload/").getPath(); } @Test @@ -31,7 +31,7 @@ public void test_fetch_upload_id_finds_improperly_formatted_file() throws IOExce @Test public void test_upload_state_store_containing_dir() throws IOException { - val pathString = getTestResourceRoot().getCanonicalPath(); + val pathString = getTestResourceRoot(); val testFile = new File(pathString); val resultFile = UploadStateStore.getContainingDir(testFile); assertThat(resultFile).isNotNull(); diff --git a/score-core/pom.xml b/score-core/pom.xml index 6f8c6910..536e4b86 100644 --- a/score-core/pom.xml +++ b/score-core/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 ../pom.xml diff --git a/score-fs/pom.xml b/score-fs/pom.xml index e8c4d1f0..ba1a32b2 100644 --- a/score-fs/pom.xml +++ b/score-fs/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 ../pom.xml diff --git a/score-server/pom.xml b/score-server/pom.xml index d744470d..7e3fd5d3 100644 --- a/score-server/pom.xml +++ b/score-server/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 ../pom.xml diff --git a/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java b/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java index 17e235d0..c37fbc32 100644 --- a/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java +++ b/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java @@ -31,6 +31,7 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import lombok.Setter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.beans.factory.annotation.Autowired; @@ -40,6 +41,7 @@ import org.springframework.stereotype.Service; import java.net.URISyntaxException; +import java.util.Base64; import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -99,8 +101,8 @@ public ObjectSpecification download(String objectId, long offset, long length, b } fillPartUrls(objectId, parts); - return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, blob.getProperties() - .getContentMD5(), false); + val md5 = base64ToHexMD5(blob.getProperties().getContentMD5()); + return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, md5, false); } catch (StorageException e) { log.error("Failed to download objectId: {}, offset: {}, length: {}, forExternalUse: {}: {} ", objectId, offset, length, forExternalUse, e); @@ -146,4 +148,16 @@ public String getSentinelObject() { null); return result; } + + @SneakyThrows + static String base64ToHexMD5(String content) { + val bytes = Base64.getDecoder().decode(content); + val output = new StringBuilder(); + for (val b : bytes) { + output.append(String.format("%02x", b)); + } + + log.trace("Converted MD5 from {} to {}", content, output); + return output.toString(); + } } diff --git a/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java new file mode 100644 index 00000000..4d91d278 --- /dev/null +++ b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java @@ -0,0 +1,17 @@ +package bio.overture.score.server.repository.azure; + +import junit.framework.TestCase; +import lombok.val; +import org.junit.Test; + +public class AzureDownloadServiceTest extends TestCase { + + @Test + public void testContentMD5Conversion() { + val contentMD5 = "iC9XnDziOQvKwyeOSUfKIg=="; + val converted = AzureDownloadService.base64ToHexMD5(contentMD5); + + assertEquals("882f579c3ce2390bcac3278e4947ca22", converted); + } + +} diff --git a/score-test/pom.xml b/score-test/pom.xml index 5a972173..5d6ad601 100644 --- a/score-test/pom.xml +++ b/score-test/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0 ../pom.xml