diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 79ee81f0884..19ddda25b5a 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -2032,6 +2032,7 @@ This section contains reference information about each of the configuration prop |`fhirServer/bulkdata/core/cos/requestTimeout`|number|The request timeout in second for the COS client| |`fhirServer/bulkdata/core/cos/socketTimeout`|number|The socket timeout in second for the COS client| |`fhirServer/bulkdata/core/cos/useServerTruststore`|boolean|If the COS Client should use the IBM FHIR Server's TrustStore to access S3/IBMCOS service | +|`fhirServer/bulkdata/core/cos/presignedExpiry`|number|The time in seconds of the presigned download URL; must be using HMAC auth| |`fhirServer/bulkdata/core/file/writeTriggerSizeMB`|number|The size, in megabytes, at which to write the buffer to file.| |`fhirServer/bulkdata/core/file/sizeThresholdMB`|number|The size, in megabytes, at which to finish writing a given file. Use `0` to indicate that all resources of a given type should be written to a single file.| |`fhirServer/bulkdata/core/file/resourceCountThreshold`|number|The number of resources at which to finish writing a given file. The actual number of resources written to a single file may be slightly above this number, dependent on the configured page size. Use `0` to indicate that there is no limit to the number of resources to be written to a single file.| @@ -2040,7 +2041,7 @@ This section contains reference information about each of the configuration prop |`fhirServer/bulkdata/core/maxPartitions`|number| The maximum number of simultaneous partitions that are processed per Export and Import | |`fhirServer/bulkdata/core/maxInputs`|number| The number of inputs allowed for $import | |`fhirServer/bulkdata/core/iamEndpoint`|string| Override the system's IAM endpoint | -|`fhirServer/bulkdata/core/fastTxTimeout`|number| Time timeout for the fast implementations transaction | +|`fhirServer/bulkdata/core/maxChunkReadTime`|string| Max time in milliseconds to read during a bulkdata export without type filters. The time should be three quarters of the transactionManager timeout (often the FHIR_TRANSACTION_MANAGER_TIMEOUT value). Note, this value is a string representation of a long value.| |`fhirServer/bulkdata/storageProviders//type`|string|The type of storageProvider aws-s3, ibm-cos, file, https | |`fhirServer/bulkdata/storageProviders//bucketName`|string| Object store bucket name | |`fhirServer/bulkdata/storageProviders//location`|string|Object store location | @@ -2152,11 +2153,12 @@ This section contains reference information about each of the configuration prop |`fhirServer/bulkdata/core/cos/requestTimeout`|120| |`fhirServer/bulkdata/core/cos/socketTimeout`|120| |`fhirServer/bulkdata/core/cos/useServerTruststore`|false| +|`fhirServer/bulkdata/core/cos/presignedExpiry`|86400| |`fhirServer/bulkdata/core/pageSize`|1000| |`fhirServer/bulkdata/core/maxPartitions`|5| |`fhirServer/bulkdata/core/maxInputs`|5| |`fhirServer/bulkdata/core/iamEndpoint`|https://iam.cloud.ibm.com/oidc/token| -|`fhirServer/bulkdata/core/fastTxTimeout`|90000| +|`fhirServer/bulkdata/core/maxChunkReadTime`|90000| |`fhirServer/bulkdata/storageProviders//disableBaseUrlValidation`|false| |`fhirServer/bulkdata/storageProviders//exportPublic`|false| |`fhirServer/bulkdata/storageProviders//enableParquet`|false| @@ -2264,6 +2266,7 @@ must restart the server for that change to take effect. |`fhirServer/bulkdata/core/cos/requestTimeout`|N|N| |`fhirServer/bulkdata/core/cos/socketTimeout`|N|N| |`fhirServer/bulkdata/core/cos/useServerTruststore`|Y|Y| +|`fhirServer/bulkdata/core/cos/presignedExpiry`|Y|Y| |`fhirServer/bulkdata/core/batchIdEncryptionKey`|Y|N| |`fhirServer/bulkdata/core/pageSize`|Y|Y| |`fhirServer/bulkdata/core/maxPartitions`|Y|Y| diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/fast/ResourcePayloadReader.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/fast/ResourcePayloadReader.java index c45ddb07077..ed241b660b0 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/fast/ResourcePayloadReader.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/fast/ResourcePayloadReader.java @@ -139,9 +139,8 @@ public class ResourcePayloadReader extends AbstractItemReader { private static final char NDJSON_LINE_SEPARATOR = '\n'; - // Assume for now we have a transaction timeout of 120s, but need to give the upload some time to work - // TODO configuration - private long txTimeoutMillis = 90000L; + // We need to give the upload some time to work + private long txTimeoutMillis = ConfigurationFactory.getInstance().getCoreFastMaxReadTimeout(); // The nanoTime after which we want to stop processing to avoid a Liberty transaction timeout private long txEndTime; diff --git a/fhir-server/liberty-config/config/default/fhir-server-config.json b/fhir-server/liberty-config/config/default/fhir-server-config.json index 693b0623dad..f015e098f29 100644 --- a/fhir-server/liberty-config/config/default/fhir-server-config.json +++ b/fhir-server/liberty-config/config/default/fhir-server-config.json @@ -148,7 +148,8 @@ "partUploadTriggerSizeMB": 10, "objectSizeThresholdMB": 200, "objectResourceCountThreshold": 200000, - "useServerTruststore": true + "useServerTruststore": true, + "presignedExpiry": 86400 }, "file" : { "writeTriggerSizeMB": 1, @@ -159,6 +160,7 @@ "batchIdEncryptionKey": "change-password", "maxPartitions": 3, "maxInputs": 5, + "maxChunkReadTime": "90000", "systemExportImpl": "fast" }, "storageProviders": { diff --git a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/ConfigurationAdapter.java b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/ConfigurationAdapter.java index 4b26f339bea..a206f3c2ca4 100644 --- a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/ConfigurationAdapter.java +++ b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/ConfigurationAdapter.java @@ -216,13 +216,11 @@ public interface ConfigurationAdapter { String getCoreIamEndpoint(); /** - * get the tx for the fast endpoint - * - * @implNote System value. + * get the number ms to read payloads from the persistence layer before stopping to checkpoint * * @return */ - int getCoreFastTxTimeout(); + long getCoreFastMaxReadTimeout(); /** * gets the StorageProvider type which aligns with the StorageType @@ -468,4 +466,10 @@ public interface ConfigurationAdapter { * @return */ int getImportInflyRateNumberOfFhirResources(String provider); -} \ No newline at end of file + + /** + * the expiry time of the generated presigned urls. + * @return + */ + int getPresignedUrlExpiry(); +} diff --git a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java index 4dde0669109..930a0d16fed 100644 --- a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java +++ b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java @@ -265,8 +265,8 @@ public String getCoreIamEndpoint() { } @Override - public int getCoreFastTxTimeout() { - return FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/fastTxTimeout", MAX_PARTITIONPROCESSING_THREADNUMBER); + public long getCoreFastMaxReadTimeout() { + return Long.parseLong(FHIRConfigHelper.getStringProperty("fhirServer/bulkdata/core/maxChunkReadTime", "90000")); } @Override @@ -317,4 +317,10 @@ public int getImportNumberOfFhirResourcesPerRead(String provider) { public int getImportInflyRateNumberOfFhirResources(String provider) { return IMPORT_INFLY_RATE_NUMOFFHIRRESOURCES; } -} \ No newline at end of file + + @Override + public int getPresignedUrlExpiry() { + int expirySeconds = FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/cos/presignedExpiry", 86400); + return Math.max(1, expirySeconds); + } +} diff --git a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/model/url/DownloadUrl.java b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/model/url/DownloadUrl.java index 241b4c68e42..a2a5f27ae38 100644 --- a/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/model/url/DownloadUrl.java +++ b/operation/fhir-operation-bulkdata/src/main/java/com/ibm/fhir/operation/bulkdata/model/url/DownloadUrl.java @@ -19,6 +19,8 @@ import javax.crypto.spec.SecretKeySpec; import javax.ws.rs.HttpMethod; +import com.ibm.fhir.operation.bulkdata.config.ConfigurationFactory; + /** * Based on the IBM Cloud Documentation * @@ -29,7 +31,6 @@ public class DownloadUrl { private static final Logger logger = Logger.getLogger(DownloadUrl.class.getName()); private static final String HTTP_METHOD = HttpMethod.GET; - private static final String EXPIRY_SECONDS = String.valueOf(86400); private static final MessageDigest digest = createSigningDigest(); @@ -110,10 +111,12 @@ public String getSignedUrl() throws Exception { String datestamp = time.format(DateTimeFormatter.ofPattern("yyyyMMdd")); String timestamp = datestamp + "T" + time.format(DateTimeFormatter.ofPattern("HHmmss")) + "Z"; + String expirySeconds = String.valueOf(ConfigurationFactory.getInstance().getPresignedUrlExpiry()); + String standardizedQuerystring = "X-Amz-Algorithm=AWS4-HMAC-SHA256" + "&X-Amz-Credential=" + URLEncoder.encode(accessKey + "/" + datestamp + "/" + region + "/s3/aws4_request", StandardCharsets.UTF_8.toString()) + "&X-Amz-Date=" + timestamp + - "&X-Amz-Expires=" + EXPIRY_SECONDS + + "&X-Amz-Expires=" + expirySeconds + "&X-Amz-SignedHeaders=host"; String standardizedResource = "/" + bucketName + "/" + cosBucketPathPrefix + "/"+ objectKey;