Auto sync storage profiles #378 (#382)

* added server endpoint to get profiles

* change to switch score-client storage implementations

* cleared azure and s3 related entries app.yml

* some bug fixes and enhancements

* updated comment

* added test profile.

* added test profile.

* refactored code based on review comments - BaseController now returns a single profile value. The actual profile name and the profile value returned by the api are now different.

* debug logging removed

* replaced profile value

* updated readme

* updated readme

* added a test profile

* review changes
- Storage profile values now come from an enum in score-core
- test configuration created to mock storage profile bean
- users will be able to provide a default profile value when working with old score-server instances

* users will be able to provide a default profile value when working with old score-server instances

* config change

* added a test config in score server

---------

Co-authored-by: UmmulkiramR <urangwala@oicr.on.ca>

score-client/README.md

@@ -78,5 +78,10 @@ To enable logging of request bodies and headers, append the following to the com
 `--logging.level.org.apache.http=DEBUG`
 
 ## Azure
-For azure support, use the `azure` profile. Block sizes for uploads
+
+Score-client dynamically loads storage profile specific implementations (currently S3 or Azure) based on the profile score-server is running on.
+It does this by calling an endpoint on score-server to fetch the active storage profiles.
+For older score-server versions where the profile endpoint isn't available, the score-client will use the default (S3) storage profile implementation.
+This eliminates the need for users to ensure that score-client is running on a profile compatible with score-server.
+
 can be configured with the `azure.blockSize` property. 

score-client/src/main/java/bio/overture/score/client/command/UploadCommand.java

@@ -23,14 +23,17 @@
 import bio.overture.score.client.manifest.ManifestService;
 import bio.overture.score.client.manifest.UploadManifest;
 import bio.overture.score.client.upload.UploadService;
+import bio.overture.score.client.util.BeanUtil;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -64,9 +67,19 @@ public class UploadCommand extends RepositoryAccessCommand {
    */
   @Autowired
   private ManifestService manifestService;
-  @Autowired
   private UploadService uploader;
 
+  @Autowired
+  ApplicationContext appContext;
+
+  @Autowired
+  private BeanUtil beanUtil;
+
+  @PostConstruct
+  public void initializeStorageProfile() throws Exception{
+    uploader = (UploadService) beanUtil.getBeanForProfile(UploadService.class);
+  }
+
   @Override
   public int execute() throws Exception {
     checkParameter(objectId != null || manifestResource != null, "One of --object-id or --manifest must be specified");

score-client/src/main/java/bio/overture/score/client/config/AzureConfig.java

@@ -19,24 +19,19 @@
 
 import bio.overture.score.client.upload.UploadService;
 import bio.overture.score.client.upload.azure.AzureUploadService;
+import bio.overture.score.client.upload.s3.S3UploadService;
 import bio.overture.score.client.util.AzurePresignedUrlValidator;
 import bio.overture.score.client.util.PresignedUrlValidator;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 
+// Note: Name the beans as profile-name+class-type to be able to dynamically fetch beans at runtime based on score-server profiles
 @Configuration
-@Profile("azure")
 public class AzureConfig {
 
   @Bean
-  public UploadService uploadService() {
-    return new AzureUploadService();
-  }
-
+  public UploadService azUploadService(){ return new AzureUploadService();}
   @Bean
-  public PresignedUrlValidator urlValidator() {
-    return new AzurePresignedUrlValidator();
-  }
-
+  public AzurePresignedUrlValidator azPresignedUrlValidator(){return new AzurePresignedUrlValidator();}
 }

score-client/src/main/java/bio/overture/score/client/config/ProfileConfig.java

@@ -0,0 +1,72 @@
+package bio.overture.score.client.config;
+
+import bio.overture.score.client.exception.NotRetryableException;
+import bio.overture.score.core.model.StorageProfiles;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+@Slf4j
+@ConditionalOnProperty(value="isTest", havingValue="false")
+public class ProfileConfig {
+
+  @Autowired
+  private RestTemplate serviceTemplate;
+
+  @Value("${storage.url}")
+  @NonNull
+  private String endpoint;
+
+  @Autowired
+  @Value("${defaultProfile:collaboratory}")
+  private String defaultProfile;
+
+  @Autowired
+  @Value("${isTest}")
+  private boolean isTest = false;
+
+  @Qualifier("clientVersion")
+  @Autowired
+  @NonNull
+  String clientVersion;
+
+  @Bean
+  public String storageProfile(){
+    String profile =  getStorageProfile();
+    return profile;
+  }
+
+  private String getStorageProfile() {
+    log.debug("get profile endpoint: "+endpoint);
+  try{
+    String storageProfile = serviceTemplate.exchange(endpoint + "/profile", HttpMethod.GET, defaultEntity(), String.class).getBody();
+    return storageProfile;
+  }catch(NotRetryableException nre ){
+    log.error("received exception when getting profiles: " + nre.getMessage());
+  }
+  return StorageProfiles.getProfileValue(defaultProfile);
+  }
+
+
+  private HttpEntity<Object> defaultEntity() {
+    return new HttpEntity<Object>(defaultHeaders());
+  }
+
+  private HttpHeaders defaultHeaders() {
+    val requestHeaders = new HttpHeaders();
+    requestHeaders.add(HttpHeaders.USER_AGENT, clientVersion);
+    return requestHeaders;
+  }
+
+}

score-client/src/main/java/bio/overture/score/client/config/S3Config.java

@@ -19,24 +19,21 @@
 
 import bio.overture.score.client.upload.UploadService;
 import bio.overture.score.client.upload.s3.S3UploadService;
+import bio.overture.score.client.util.AzurePresignedUrlValidator;
 import bio.overture.score.client.util.PresignedUrlValidator;
 import bio.overture.score.client.util.S3PresignedUrlValidator;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 
+// Note: Name the beans as profile-name+class-type to be able to dynamically fetch beans at runtime based on score-server profiles
 @Configuration
-@Profile({ "!azure" })
 public class S3Config {
 
   @Bean
-  public UploadService uploadService() {
-    return new S3UploadService();
-  }
-
+  public UploadService s3UploadService(){return new S3UploadService();}
   @Bean
-  public PresignedUrlValidator urlValidator() {
+  public S3PresignedUrlValidator s3PresignedUrlValidator(){
     return new S3PresignedUrlValidator();
   }
-
 }

score-client/src/main/java/bio/overture/score/client/download/DownloadStateStore.java

@@ -19,6 +19,7 @@
 
 import bio.overture.score.client.exception.NotRetryableException;
 import bio.overture.score.client.state.TransferState;
+import bio.overture.score.client.util.BeanUtil;
 import bio.overture.score.client.util.PresignedUrlValidator;
 import bio.overture.score.core.model.ObjectSpecification;
 import bio.overture.score.core.model.Part;
@@ -27,6 +28,7 @@
 import lombok.val;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import javax.annotation.PostConstruct;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
@@ -35,11 +37,16 @@
 @Slf4j
 public class DownloadStateStore extends TransferState {
 
-  @Autowired
   private PresignedUrlValidator urlValidator;
-
+  @Autowired
+  private BeanUtil beanUtil;
   private static final ObjectMapper MAPPER = new ObjectMapper();
 
+  @PostConstruct
+  public void initializeStorageProfile() throws Exception{
+    urlValidator = (PresignedUrlValidator) beanUtil.getBeanForProfile(PresignedUrlValidator.class);
+  }
+
   public void init(File stateDir, ObjectSpecification spec) {
     log.debug("Download Specification : {}", spec);
     try {

score-client/src/main/java/bio/overture/score/client/upload/azure/AzureUploadService.java

@@ -54,6 +54,7 @@ public class AzureUploadService implements UploadService {
 
   @Override
   public void upload(File file, String objectId, String md5, boolean redo) throws IOException {
+    log.debug("Initiating azure upload");
     // Get object specification from server
     val spec = storageService.initiateUpload(objectId, file.length(), redo, md5);
 

score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java

@@ -80,6 +80,7 @@ public void setup() {
    */
   @Override
   public void upload(File file, String objectId, String md5, final boolean redo) throws IOException {
+    log.debug("Initiating S3 upload");
     boolean tryAgain = redo;
     for (int retry = 0; retry < retryNumber; retry++)
       try {

score-client/src/main/java/bio/overture/score/client/util/BeanUtil.java

@@ -0,0 +1,31 @@
+package bio.overture.score.client.util;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+// utility class to inject beans dynamically at runtime
+
+@Component
+public class BeanUtil<T> {
+
+  @Autowired
+  ApplicationContext appContext;
+
+  //this method injects beans dynamically based on score-server active profile
+   public Object getBeanForProfile(Class cl){
+     String profile = appContext.getBean("storageProfile", String.class);
+
+     Set<String> beans = appContext.getBeansOfType(cl).keySet();
+     return appContext.getBean(beans.stream()
+         .filter(s -> s.equals(profile+cl.getSimpleName()))
+         .collect(Collectors.toList())
+         .get(0), cl);
+  }
+}

score-client/src/main/resources/application.yml

@@ -76,42 +76,9 @@ mount:
 token:
   public-key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlvqX7PDezX7BC0yc14zwJw2qgs5KDyZZvEeNowHCgHLcCtCVZ2JIx+ccUPp1FVudfZpLoboZ3RjT+6oOPHpNilXqUZfT81acCVHhvyJe7FNxdUWKpIIVKM/8bJj1pVkst0YHLBJ3KojZYJCMNUxUkyHM+yGrn9tmSdFBmwJnBLTkSxOrS2bvuZvkjERnoQJCoTm2XBKoCOzEuVPjIeB2KiFcNqiZwhjnCSQGPmg7fyJfQt1vMl+8RST3IXxphf8KKaWMl0rTXQhIHYL23P/tvC2lpJgx0rzgnxkAgFSrCxLstIklFAAg+j7ykivOyynIlCfXGv4cPTq5UqvIB8MBRQIDAQAB
 
----
-
-###############################################################################
-# Profile - "aws"
-###############################################################################
-
-spring.profiles: aws
-
-storage:
-  url: https://virginia.cloud.icgc.org
-
-metadata:
-  url: https://virginia.song.icgc.org
-  ssl:
-    enabled: true
-
-client:
-  ssl:
-    custom: false
-
----
-
-###############################################################################
-# Profile - "azure"
-###############################################################################
-
-spring.profiles: azure
 
-storage:
-  url: https://score.azure.icgc.overture.bio
-
-metadata:
-  url: https://song.azure.icgc.overture.bio
-
-azure:
-  blockSize: 104857600
+isTest: false
+defaultProfile: collaboratory
 
 ---
 
@@ -132,25 +99,6 @@ client:
 
 ---
 
-###############################################################################
-# Profile - "collab"
-###############################################################################
-
-spring.profiles: collab
-
-# Storage server
-storage:
-  url: https://storage.cancercollaboratory.org
-
-metadata:
-  url: https://song.cancercollaboratory.org
-
-client:    
-  ssl:
-    custom: false
-
----
-
 ###############################################################################
 # Profile - "oicr"
 ###############################################################################
@@ -174,3 +122,8 @@ logging:
     springframework.retry: DEBUG
     springframework.web: DEBUG
     com.amazonaws.services: DEBUG
+
+---
+###############################################################################
+# Profile - "debug"
+###############################################################################

score-client/src/test/java/bio/overture/score/client/ClientMainTest.java

@@ -23,6 +23,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.springframework.test.context.ActiveProfiles;
 
 import java.io.File;
 import java.util.UUID;
@@ -31,6 +32,7 @@
 import static org.junit.Assert.assertTrue;
 
 @Slf4j
+@ActiveProfiles("test")
 public class ClientMainTest extends AbstractClientMainTest {
 
   @Rule

score-client/src/test/java/bio/overture/score/client/config/TestProfileConfig.java

@@ -0,0 +1,26 @@
+package bio.overture.score.client.config;
+
+import bio.overture.score.client.exception.NotRetryableException;
+import bio.overture.score.core.model.StorageProfiles;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.http.HttpMethod;
+import org.springframework.test.context.ContextConfiguration;
+
+@TestConfiguration
+public class TestProfileConfig {
+
+    @Autowired
+    @Value("${defaultProfile:collaboratory}")
+    private String defaultProfile;
+
+    @Bean
+    public String storageProfile(){
+        return StorageProfiles.getProfileValue(defaultProfile);
+    }
+
+}