Merge pull request #2171 from IBM/issue-2150

configure the maxChunkReadTime for Fast Export and configurable timeout for presigned URL #2150

docs/src/pages/guides/FHIRServerUsersGuide.md

@@ -2032,6 +2032,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/bulkdata/core/cos/requestTimeout`|number|The request timeout in second for the COS client|
 |`fhirServer/bulkdata/core/cos/socketTimeout`|number|The socket timeout in second for the COS client|
 |`fhirServer/bulkdata/core/cos/useServerTruststore`|boolean|If the COS Client should use the IBM FHIR Server's TrustStore to access S3/IBMCOS service |
+|`fhirServer/bulkdata/core/cos/presignedExpiry`|number|The time in seconds of the presigned download URL; must be using HMAC auth|
 |`fhirServer/bulkdata/core/file/writeTriggerSizeMB`|number|The size, in megabytes, at which to write the buffer to file.|
 |`fhirServer/bulkdata/core/file/sizeThresholdMB`|number|The size, in megabytes, at which to finish writing a given file. Use `0` to indicate that all resources of a given type should be written to a single file.|
 |`fhirServer/bulkdata/core/file/resourceCountThreshold`|number|The number of resources at which to finish writing a given file. The actual number of resources written to a single file may be slightly above this number, dependent on the configured page size. Use `0` to indicate that there is no limit to the number of resources to be written to a single file.|
@@ -2040,7 +2041,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/bulkdata/core/maxPartitions`|number| The maximum number of simultaneous partitions that are processed per Export and Import |
 |`fhirServer/bulkdata/core/maxInputs`|number| The number of inputs allowed for $import |
 |`fhirServer/bulkdata/core/iamEndpoint`|string| Override the system's IAM endpoint |
-|`fhirServer/bulkdata/core/fastTxTimeout`|number| Time timeout for the fast implementations transaction |
+|`fhirServer/bulkdata/core/maxChunkReadTime`|string| Max time in milliseconds to read during a bulkdata export without type filters. The time should be three quarters of the transactionManager timeout (often the FHIR_TRANSACTION_MANAGER_TIMEOUT value). Note, this value is a string representation of a long value.|
 |`fhirServer/bulkdata/storageProviders/<source>/type`|string|The type of storageProvider aws-s3, ibm-cos, file, https |
 |`fhirServer/bulkdata/storageProviders/<source>/bucketName`|string| Object store bucket name |
 |`fhirServer/bulkdata/storageProviders/<source>/location`|string|Object store location |
@@ -2152,11 +2153,12 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/bulkdata/core/cos/requestTimeout`|120|
 |`fhirServer/bulkdata/core/cos/socketTimeout`|120|
 |`fhirServer/bulkdata/core/cos/useServerTruststore`|false|
+|`fhirServer/bulkdata/core/cos/presignedExpiry`|86400|
 |`fhirServer/bulkdata/core/pageSize`|1000|
 |`fhirServer/bulkdata/core/maxPartitions`|5|
 |`fhirServer/bulkdata/core/maxInputs`|5|
 |`fhirServer/bulkdata/core/iamEndpoint`|https://iam.cloud.ibm.com/oidc/token|
-|`fhirServer/bulkdata/core/fastTxTimeout`|90000|
+|`fhirServer/bulkdata/core/maxChunkReadTime`|90000|
 |`fhirServer/bulkdata/storageProviders/<source>/disableBaseUrlValidation`|false|
 |`fhirServer/bulkdata/storageProviders/<source>/exportPublic`|false|
 |`fhirServer/bulkdata/storageProviders/<source>/enableParquet`|false|
@@ -2264,6 +2266,7 @@ must restart the server for that change to take effect.
 |`fhirServer/bulkdata/core/cos/requestTimeout`|N|N|
 |`fhirServer/bulkdata/core/cos/socketTimeout`|N|N|
 |`fhirServer/bulkdata/core/cos/useServerTruststore`|Y|Y|
+|`fhirServer/bulkdata/core/cos/presignedExpiry`|Y|Y|
 |`fhirServer/bulkdata/core/batchIdEncryptionKey`|Y|N|
 |`fhirServer/bulkdata/core/pageSize`|Y|Y|
 |`fhirServer/bulkdata/core/maxPartitions`|Y|Y|

fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/fast/ResourcePayloadReader.java

@@ -139,9 +139,8 @@ public class ResourcePayloadReader extends AbstractItemReader {
 
     private static final char NDJSON_LINE_SEPARATOR = '\n';
 
-    // Assume for now we have a transaction timeout of 120s, but need to give the upload some time to work
-    // TODO configuration
-    private long txTimeoutMillis = 90000L;
+    // We need to give the upload some time to work
+    private long txTimeoutMillis = ConfigurationFactory.getInstance().getCoreFastMaxReadTimeout();
 
     // The nanoTime after which we want to stop processing to avoid a Liberty transaction timeout
     private long txEndTime;

fhir-server/liberty-config/config/default/fhir-server-config.json

@@ -148,7 +148,8 @@
                     "partUploadTriggerSizeMB": 10,
                     "objectSizeThresholdMB": 200,
                     "objectResourceCountThreshold": 200000,
-                    "useServerTruststore": true
+                    "useServerTruststore": true,
+                    "presignedExpiry": 86400
                 },
                 "file" : { 
                     "writeTriggerSizeMB": 1,
@@ -159,6 +160,7 @@
                 "batchIdEncryptionKey": "change-password",
                 "maxPartitions": 3, 
                 "maxInputs": 5,
+                "maxChunkReadTime": "90000",
                 "systemExportImpl": "fast"
             },
             "storageProviders": {

operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/ConfigurationAdapter.java

@@ -216,13 +216,11 @@ public interface ConfigurationAdapter {
     String getCoreIamEndpoint();
 
     /**
-     * get the tx for the fast endpoint
-     *
-     * @implNote System value.
+     * get the number ms to read payloads from the persistence layer before stopping to checkpoint
      *
      * @return
      */
-    int getCoreFastTxTimeout();
+    long getCoreFastMaxReadTimeout();
 
     /**
      * gets the StorageProvider type which aligns with the StorageType
@@ -468,4 +466,10 @@ public interface ConfigurationAdapter {
      * @return
      */
     int getImportInflyRateNumberOfFhirResources(String provider);
-}
+
+    /**
+     * the expiry time of the generated presigned urls.
+     * @return
+     */
+    int getPresignedUrlExpiry();
+}

operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java

@@ -265,8 +265,8 @@ public String getCoreIamEndpoint() {
     }
 
     @Override
-    public int getCoreFastTxTimeout() {
-        return FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/fastTxTimeout", MAX_PARTITIONPROCESSING_THREADNUMBER);
+    public long getCoreFastMaxReadTimeout() {
+        return Long.parseLong(FHIRConfigHelper.getStringProperty("fhirServer/bulkdata/core/maxChunkReadTime", "90000"));
     }
 
     @Override
@@ -317,4 +317,10 @@ public int getImportNumberOfFhirResourcesPerRead(String provider) {
     public int getImportInflyRateNumberOfFhirResources(String provider) {
         return IMPORT_INFLY_RATE_NUMOFFHIRRESOURCES;
     }
-}
+
+    @Override
+    public int getPresignedUrlExpiry() {
+        int expirySeconds = FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/cos/presignedExpiry", 86400);
+        return Math.max(1, expirySeconds);
+    }
+}

operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/model/url/DownloadUrl.java

@@ -19,6 +19,8 @@
 import javax.crypto.spec.SecretKeySpec;
 import javax.ws.rs.HttpMethod;
 
+import com.ibm.fhir.operation.bulkdata.config.ConfigurationFactory;
+
 /**
  * Based on the IBM Cloud Documentation
  *
@@ -29,7 +31,6 @@ public class DownloadUrl {
     private static final Logger logger = Logger.getLogger(DownloadUrl.class.getName());
 
     private static final String HTTP_METHOD = HttpMethod.GET;
-    private static final String EXPIRY_SECONDS = String.valueOf(86400);
 
     private static final MessageDigest digest = createSigningDigest();
 
@@ -110,10 +111,12 @@ public String getSignedUrl() throws Exception {
         String datestamp = time.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
         String timestamp = datestamp + "T" + time.format(DateTimeFormatter.ofPattern("HHmmss")) + "Z";
 
+        String expirySeconds = String.valueOf(ConfigurationFactory.getInstance().getPresignedUrlExpiry());
+
         String standardizedQuerystring = "X-Amz-Algorithm=AWS4-HMAC-SHA256" +
                 "&X-Amz-Credential=" + URLEncoder.encode(accessKey + "/" + datestamp + "/" + region + "/s3/aws4_request", StandardCharsets.UTF_8.toString()) +
                 "&X-Amz-Date=" + timestamp +
-                "&X-Amz-Expires=" + EXPIRY_SECONDS +
+                "&X-Amz-Expires=" + expirySeconds +
                 "&X-Amz-SignedHeaders=host";
 
         String standardizedResource = "/" + bucketName + "/" + cosBucketPathPrefix + "/"+ objectKey;