Rc/5.4.0

AZURE.md

@@ -0,0 +1,44 @@
+# Azure Documentation
+
+## Server Setup
+
+### Storage Container Setup
+Assuming you have a storage account already created inside a resource group
+in Azure, create a container called `data`. 
+
+Upload a sentinel object into the container. This can be a small text file that
+score will use to validate connectivity to the object storage at request time. 
+
+Create access policies on the storage container. One for read operations and one
+for write operations. 
+Example:
+![Access Policies](./docs/img/azure_policies.png)
+
+Configure the score server either in the config file or env vars:
+
+```yml
+object:
+  sentinel: <object_name> # name of the sentinel object you upoaded
+
+azure:
+  endpointProtocol: https
+  accountName: <storage_account_name>
+  accountKey: <storage_account_secret_key>
+
+bucket:
+  name.object: data # container name
+  policy.upload: <write_policy> # name of the policy for the write/add/modify operations
+  policy.download: <read_policy> # name of policy for the read/list operations
+
+download:
+  partsize: 250000000 # safe default
+```
+
+Start the server with the following run profiles enabled: `prod,secure,azure`
+
+## Client Setup
+Follow the regular client setup of access token and storage and metadata urls.
+However when running the client use azure pofile option:
+```bash
+$ bin/score-client --profile azure ...
+```

pom.xml

@@ -20,7 +20,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
 
   <groupId>bio.overture</groupId>
   <artifactId>score</artifactId>
-  <version>5.3.0</version>
+  <version>5.4.0</version>
   <packaging>pom</packaging>
 
   <name>${project.artifactId}</name>

score-client/pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>bio.overture</groupId>
     <artifactId>score</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java

@@ -41,34 +41,24 @@ public class UploadStateStore extends TransferState {
    * /path/to/.<object-id>/uploadid upload state file
    */
 
-  /*
-   * We are using the directory that the upload file is currently located in. Requires read/write access to directory
-   * Otherwise, we need to have user specify a working directory for upload - which is a bit too counter-intuitive.
-   */
   @SneakyThrows
-  public static File getContainingDir(@NonNull final File uploadFile) {
-    val parentPath = Optional.ofNullable(uploadFile.getParent());
-    if (parentPath.isPresent()) {
-      return new File(parentPath.get());
-    }
-    return null;
+  public static String getContainingDir(@NonNull final File uploadFile) {
+    return uploadFile.getParent();
   }
 
   /**
    * Write upload-id of current upload into state directory (hidden directory next to file being uploaded)
-   * @param uploadFile - File we want to upload. Used to determine path to create temporary upload id directory
+   * @param uploadStateDir - Path to create temporary upload id directory
    * @param spec
-   * @param force
    */
-  public static void create(@NonNull File uploadFile, @NonNull ObjectSpecification spec, boolean force) {
-    val filePath = getContainingDir(uploadFile);
-
+  public static void create(@NonNull String uploadStateDir, @NonNull ObjectSpecification spec) throws NotRetryableException {
     try {
-      val uploadStateDir = getObjectStateDir(filePath, spec.getObjectId());
+      val objectStatePath = getObjectStatePath(uploadStateDir, spec.getObjectId());
+      val objectStateDir = new File(objectStatePath);
 
-      removeDir(uploadStateDir, true);
+      removeDir(objectStateDir, true);
 
-      val uploadIdFile = new File(uploadStateDir, getStateName());
+      val uploadIdFile = new File(objectStateDir, getStateName());
       try (PrintWriter out = new PrintWriter(uploadIdFile, StandardCharsets.UTF_8.name())) {
         out.println(spec.getUploadId());
       }
@@ -82,10 +72,10 @@ protected static String getStateName() {
     return "uploadId";
   }
 
-  public static Optional<String> fetchUploadId(@NonNull File uploadFile, @NonNull String objectId) {
-    Optional<String> result = Optional.ofNullable(null);
-    val uploadStateDir = getObjectStateDir(getContainingDir(uploadFile), objectId);
-    val uploadIdFile = new File(uploadStateDir, getStateName());
+  public static Optional<String> fetchUploadId(@NonNull String uploadStateDir, @NonNull String objectId) {
+    Optional<String> result = Optional.empty();
+    val objectStatePath = getObjectStatePath(uploadStateDir, objectId);
+    val uploadIdFile = new File(objectStatePath, getStateName());
 
     if (uploadIdFile.exists()) {
       try (val reader =
@@ -103,4 +93,13 @@ public static Optional<String> fetchUploadId(@NonNull File uploadFile, @NonNull
     }
     return result;
   }
+
+  public static void close(@NonNull String uploadStateDir, @NonNull String objectId) throws IOException {
+    val dirToDelete = new File(getObjectStatePath(uploadStateDir, objectId));
+    deleteDirectoryIfExist(dirToDelete);
+  }
+
+  private static String getObjectStatePath(@NonNull String uploadStateDir, @NonNull String objectId) {
+    return uploadStateDir +  "/." + objectId;
+  }
 }

score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java

@@ -52,6 +52,8 @@ public class S3UploadService implements UploadService {
   private boolean quiet;
   @Value("${storage.retryNumber}")
   private int retryNumber;
+  @Value("${client.uploadStateDir}")
+  private String uploadStateDir;
 
   /**
    * Dependencies.
@@ -104,16 +106,18 @@ private void startUpload(File file, String objectId, String md5, boolean overwri
     ObjectSpecification spec = null;
     try {
       spec = storageService.initiateUpload(objectId, file.length(), overwrite, md5);
+
+      // Delete if already present
+      if (overwrite) {
+        UploadStateStore.create(getUploadStateDir(file), spec);
+      }
     } catch (NotRetryableException e) {
       // A NotRetryable exception during initiateUpload should just end whole process
       // a bit of a sleazy hack. Should only be thrown when the Metadata service informs us the supplied
       // object id was never registered/does not exist in Metadata repo
-      throw new NotResumableException(e);
-    }
 
-    // Delete if already present
-    if (overwrite) {
-      UploadStateStore.create(file, spec, false);
+      // being unable to create file for upload state should also end whole process
+      throw new NotResumableException(e);
     }
 
     val progress = new Progress(terminal, quiet, spec.getParts().size(), 0);
@@ -144,7 +148,7 @@ private UploadProgress checkProgress(File uploadFile, String objectId) {
 
     // See if there is already an upload in progress for this object id. Fetch upload id and send if present. If
     // missing, send null
-    val uploadId = UploadStateStore.fetchUploadId(uploadFile, objectId);
+    val uploadId = UploadStateStore.fetchUploadId(getUploadStateDir(uploadFile), objectId);
     val progress = storageService.getProgress(objectId, uploadFile.length());
 
     // Compare upload id's
@@ -231,7 +235,14 @@ public boolean isObjectExist(String objectId) throws IOException {
   }
 
   private void cleanupState(File uploadFile, String objectId) throws IOException {
-    UploadStateStore.close(UploadStateStore.getContainingDir(uploadFile), objectId);
+    UploadStateStore.close(getUploadStateDir(uploadFile), objectId);
   }
 
+  private String getUploadStateDir(File uploadFile) {
+    if (!uploadStateDir.isEmpty()) {
+      return uploadStateDir;
+    } else {
+      return UploadStateStore.getContainingDir(uploadFile);
+    }
+  }
 }

score-client/src/main/resources/application.yml

@@ -29,6 +29,9 @@ client:
   accessToken: "${accessToken:}" # Alias for backwards compatibility
   connectTimeoutSeconds: 60
   readTimeoutSeconds: 60
+  # path of dir with WRITE access for score client upload state files
+  # if empty uses parent dir of current file to upload
+  uploadStateDir: ""
 
   ssl:
     custom: false

score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java

@@ -11,8 +11,8 @@
 
 public class UploadStateStoreTests {
 
-  File getTestResourceRoot() {
-    return new File(getClass().getClassLoader().getResource("fixtures/upload/placeholder-upload-file.txt").getFile());
+  String getTestResourceRoot() {
+    return getClass().getClassLoader().getResource("fixtures/upload/").getPath();
   }
 
   @Test
@@ -31,7 +31,7 @@ public void test_fetch_upload_id_finds_improperly_formatted_file() throws IOExce
 
   @Test
   public void test_upload_state_store_containing_dir() throws IOException {
-    val pathString = getTestResourceRoot().getCanonicalPath();
+    val pathString = getTestResourceRoot();
     val testFile = new File(pathString);
     val resultFile = UploadStateStore.getContainingDir(testFile);
     assertThat(resultFile).isNotNull();

score-core/pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>bio.overture</groupId>
     <artifactId>score</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

score-fs/pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>bio.overture</groupId>
     <artifactId>score</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

score-server/pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>bio.overture</groupId>
     <artifactId>score</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java

@@ -31,6 +31,7 @@
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 import com.microsoft.azure.storage.blob.CloudBlockBlob;
 import lombok.Setter;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -40,6 +41,7 @@
 import org.springframework.stereotype.Service;
 
 import java.net.URISyntaxException;
+import java.util.Base64;
 import java.util.List;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -99,8 +101,8 @@ public ObjectSpecification download(String objectId, long offset, long length, b
       }
       fillPartUrls(objectId, parts);
 
-      return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, blob.getProperties()
-        .getContentMD5(), false);
+      val md5 = base64ToHexMD5(blob.getProperties().getContentMD5());
+      return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, md5, false);
     } catch (StorageException e) {
       log.error("Failed to download objectId: {}, offset: {}, length: {}, forExternalUse: {}: {} ",
         objectId, offset, length, forExternalUse, e);
@@ -146,4 +148,16 @@ public String getSentinelObject() {
       null);
     return result;
   }
+
+  @SneakyThrows
+  static String base64ToHexMD5(String content) {
+    val bytes = Base64.getDecoder().decode(content);
+    val output = new StringBuilder();
+    for (val b : bytes) {
+      output.append(String.format("%02x", b));
+    }
+
+    log.trace("Converted MD5 from {} to {}", content, output);
+    return output.toString();
+  }
 }

score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java

@@ -0,0 +1,17 @@
+package bio.overture.score.server.repository.azure;
+
+import junit.framework.TestCase;
+import lombok.val;
+import org.junit.Test;
+
+public class AzureDownloadServiceTest extends TestCase {
+
+  @Test
+  public void testContentMD5Conversion() {
+    val contentMD5 = "iC9XnDziOQvKwyeOSUfKIg==";
+    val converted = AzureDownloadService.base64ToHexMD5(contentMD5);
+
+    assertEquals("882f579c3ce2390bcac3278e4947ca22", converted);
+  }
+
+}

score-test/pom.xml

@@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S
   <parent>
     <groupId>bio.overture</groupId>
     <artifactId>score</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>