diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh index 2a057ff4..6cda17cd 100755 --- a/common/docker-entrypoint.d/00-check-for-required-env.sh +++ b/common/docker-entrypoint.d/00-check-for-required-env.sh @@ -35,6 +35,10 @@ if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then echo "Running inside an ECS task, using container credentials" elif [[ -v S3_SESSION_TOKEN ]]; then + echo "Depreciated the S3_SESSION_TOKEN! Use the environment variable of AWS_SESSION_TOKEN instead" + failed=1 + +elif [[ -v AWS_SESSION_TOKEN ]]; then echo "S3 Session token specified - not using IMDS for credentials" # b) Using Instance Metadata Service (IMDS) credentials, if IMDS is present at http://169.254.169.254. @@ -48,11 +52,32 @@ elif curl --output /dev/null --silent --head --fail --connect-timeout 2 --max-ti # Example: We are running inside an EKS cluster with IAM roles for service accounts enabled. elif [[ -v AWS_WEB_IDENTITY_TOKEN_FILE ]]; then echo "Running inside EKS with IAM roles for service accounts" + if [[ -v HOSTNAME ]]; then + echo "Depreciated the HOSTNAME! Use the environment variable of AWS_ROLE_SESSION_NAME instead" + failed=1 + fi + if [[ ! -v AWS_ROLE_SESSION_NAME ]]; then + # The default value is set as a nginx-s3-gateway unless the value is defined. + AWS_ROLE_SESSION_NAME="nginx-s3-gateway" + fi + +elif [[ -v S3_ACCESS_KEY_ID ]]; then + echo "Depreciated the S3_ACCESS_KEY_ID! Use the environment variable of AWS_ACCESS_KEY_ID instead" + failed=1 + +elif [[ -v S3_SECRET_KEY ]]; then + echo "Depreciated the S3_SECRET_KEY! Use the environment variable of AWS_SECRET_ACCESS_KEY instead" + failed=1 # If none of the options above is used, require static credentials. # See https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html. else - required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY") + required+=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") +fi + +if [[ -v S3_DEBUG ]]; then + echo "Depreciated the S3_DEBUG! Use the environment variable of DEBUG instead" + failed=1 fi for name in ${required[@]}; do @@ -101,7 +126,7 @@ if [ $failed -gt 0 ]; then fi echo "S3 Backend Environment" -echo "Access Key ID: ${S3_ACCESS_KEY_ID}" +echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" diff --git a/common/etc/nginx/include/awscredentials.js b/common/etc/nginx/include/awscredentials.js index 417a04d8..760190eb 100644 --- a/common/etc/nginx/include/awscredentials.js +++ b/common/etc/nginx/include/awscredentials.js @@ -18,6 +18,40 @@ import utils from "./utils.js"; const fs = require('fs'); +/** + * The current moment as a timestamp. This timestamp will be used across + * functions in order for there to be no variations in signatures. + * @type {Date} + */ +const NOW = new Date(); + +/** + * Constant base URI to fetch credentials together with the credentials relative URI, see + * https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html for more details. + * @type {string} + */ +const ECS_CREDENTIAL_BASE_URI = 'http://169.254.170.2'; + +/** + * @type {string} + */ +const EC2_IMDS_TOKEN_ENDPOINT = 'http://169.254.169.254/latest/api/token'; + +const EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'; + +/** + * Offset to the expiration of credentials, when they should be considered expired and refreshed. The maximum + * time here can be 5 minutes, the IMDS and ECS credentials endpoint will make sure that each returned set of credentials + * is valid for at least another 5 minutes. + * + * To make sure we always refresh the credentials instead of retrieving the same again, keep credentials until 4:30 minutes + * before they really expire. + * + * @type {number} + */ +const maxValidityOffsetMs = 4.5 * 60 * 1000; + + /** * Get the current session token from either the instance profile credential * cache or environment variables. @@ -34,24 +68,25 @@ function sessionToken(r) { } /** - * Get the instance profile credentials needed to authenticated against S3 from + * Get the instance profile credentials needed to authenticate against S3 from * a backend cache. If the credentials cannot be found, then return undefined. * @param r {Request} HTTP request object (not used, but required for NGINX configuration) * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string|null), expiration: (string|null)}} AWS instance profile credentials or undefined */ function readCredentials(r) { - // TODO: Change the generic constants naming for multiple AWS services. - if ('S3_ACCESS_KEY_ID' in process.env && 'S3_SECRET_KEY' in process.env) { - const sessionToken = 'S3_SESSION_TOKEN' in process.env ? - process.env['S3_SESSION_TOKEN'] : null; + if ('AWS_ACCESS_KEY_ID' in process.env && 'AWS_SECRET_ACCESS_KEY' in process.env) { + let sessionToken = 'AWS_SESSION_TOKEN' in process.env ? + process.env['AWS_SESSION_TOKEN'] : null; + if (sessionToken !== null && sessionToken.length === 0) { + sessionToken = null; + } return { - accessKeyId: process.env['S3_ACCESS_KEY_ID'], - secretAccessKey: process.env['S3_SECRET_KEY'], + accessKeyId: process.env['AWS_ACCESS_KEY_ID'], + secretAccessKey: process.env['AWS_SECRET_ACCESS_KEY'], sessionToken: sessionToken, expiration: null }; } - if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) { return _readCredentialsFromKeyValStore(r); } else { @@ -113,8 +148,8 @@ function _readCredentialsFromFile() { * @private */ function _credentialsTempFile() { - if (process.env['S3_CREDENTIALS_TEMP_FILE']) { - return process.env['S3_CREDENTIALS_TEMP_FILE']; + if (process.env['AWS_CREDENTIALS_TEMP_FILE']) { + return process.env['AWS_CREDENTIALS_TEMP_FILE']; } if (process.env['TMPDIR']) { return `${process.env['TMPDIR']}/credentials.json` @@ -132,7 +167,7 @@ function _credentialsTempFile() { function writeCredentials(r, credentials) { /* Do not bother writing credentials if we are running in a mode where we do not need instance credentials. */ - if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { + if (process.env['AWS_ACCESS_KEY_ID'] && process.env['AWS_SECRET_ACCESS_KEY']) { return; } @@ -171,7 +206,232 @@ function _writeCredentialsToFile(credentials) { fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials)); } +/** + * Get the credentials needed to create AWS signatures in order to authenticate + * to AWS service. If the gateway is being provided credentials via a instance + * profile credential as provided over the metadata endpoint, this function will: + * 1. Try to read the credentials from cache + * 2. Determine if the credentials are stale + * 3. If the cached credentials are missing or stale, it gets new credentials + * from the metadata endpoint. + * 4. If new credentials were pulled, it writes the credentials back to the + * cache. + * + * If the gateway is not using instance profile credentials, then this function + * quickly exits. + * + * @param r {Request} HTTP request object + * @returns {Promise} + */ +async function fetchCredentials(r) { + /* If we are not using an AWS instance profile to set our credentials we + exit quickly and don't write a credentials file. */ + if (utils.areAllEnvVarsSet(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'])) { + r.return(200); + return; + } + + let current; + + try { + current = readCredentials(r); + } catch (e) { + utils.debug_log(r, `Could not read credentials: ${e}`); + r.return(500); + return; + } + + if (current) { + // If AWS returns a Unix timestamp it will be in seconds, but in Date constructor we should provide timestamp in milliseconds + // In some situations (including EC2 and Fargate) current.expiration will be an RFC 3339 string - see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials + const expireAt = typeof current.expiration == 'number' ? current.expiration * 1000 : current.expiration + const exp = new Date(expireAt).getTime() - maxValidityOffsetMs; + if (NOW.getTime() < exp) { + r.return(200); + return; + } + } + + let credentials; + + utils.debug_log(r, 'Cached credentials are expired or not present, requesting new ones'); + + if (utils.areAllEnvVarsSet('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')) { + const relative_uri = process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || ''; + const uri = ECS_CREDENTIAL_BASE_URI + relative_uri; + try { + credentials = await _fetchEcsRoleCredentials(uri); + } catch (e) { + utils.debug_log(r, 'Could not load ECS task role credentials: ' + JSON.stringify(e)); + r.return(500); + return; + } + } + else if (utils.areAllEnvVarsSet('AWS_WEB_IDENTITY_TOKEN_FILE')) { + try { + credentials = await _fetchWebIdentityCredentials(r) + } catch (e) { + utils.debug_log(r, 'Could not assume role using web identity: ' + JSON.stringify(e)); + r.return(500); + return; + } + } else { + try { + credentials = await _fetchEC2RoleCredentials(); + } catch (e) { + utils.debug_log(r, 'Could not load EC2 task role credentials: ' + JSON.stringify(e)); + r.return(500); + return; + } + } + try { + writeCredentials(r, credentials); + } catch (e) { + utils.debug_log(r, `Could not write credentials: ${e}`); + r.return(500); + return; + } + r.return(200); +} + +/** + * Get the credentials needed to generate AWS signatures from the ECS + * (Elastic Container Service) metadata endpoint. + * + * @param credentialsUri {string} endpoint to get credentials from + * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} + * @private + */ +async function _fetchEcsRoleCredentials(credentialsUri) { + const resp = await ngx.fetch(credentialsUri); + if (!resp.ok) { + throw 'Credentials endpoint response was not ok.'; + } + const creds = await resp.json(); + + return { + accessKeyId: creds.AccessKeyId, + secretAccessKey: creds.SecretAccessKey, + sessionToken: creds.Token, + expiration: creds.Expiration, + }; +} + +/** + * Get the credentials needed to generate AWS signatures from the EC2 + * metadata endpoint. + * + * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} + * @private + */ +async function _fetchEC2RoleCredentials() { + const tokenResp = await ngx.fetch(EC2_IMDS_TOKEN_ENDPOINT, { + headers: { + 'x-aws-ec2-metadata-token-ttl-seconds': '21600', + }, + method: 'PUT', + }); + const token = await tokenResp.text(); + let resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT, { + headers: { + 'x-aws-ec2-metadata-token': token, + }, + }); + /* This _might_ get multiple possible roles in other scenarios, however, + EC2 supports attaching one role only.It should therefore be safe to take + the whole output, even given IMDS _might_ (?) be able to return multiple + roles. */ + const credName = await resp.text(); + if (credName === "") { + throw 'No credentials available for EC2 instance'; + } + resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT + credName, { + headers: { + 'x-aws-ec2-metadata-token': token, + }, + }); + const creds = await resp.json(); + + return { + accessKeyId: creds.AccessKeyId, + secretAccessKey: creds.SecretAccessKey, + sessionToken: creds.Token, + expiration: creds.Expiration, + }; +} + +/** + * Get the credentials by assuming calling AssumeRoleWithWebIdentity with the environment variable + * values ROLE_ARN, AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_SESSION_NAME + * + * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} + * @private + */ +async function _fetchWebIdentityCredentials(r) { + const arn = process.env['AWS_ROLE_ARN']; + const name = process.env['AWS_ROLE_SESSION_NAME']; + + let sts_endpoint = process.env['STS_ENDPOINT']; + if (!sts_endpoint) { + /* On EKS, the ServiceAccount can be annotated with + 'eks.amazonaws.com/sts-regional-endpoints' to control + the usage of regional endpoints. We are using the same standard + environment variable here as the AWS SDK. This is with the exception + of replacing the value `legacy` with `global` to match what EKS sets + the variable to. + See: https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html + See: https://docs.aws.amazon.com/eks/latest/userguide/configure-sts-endpoint.html */ + const sts_regional = process.env['AWS_STS_REGIONAL_ENDPOINTS'] || 'global'; + if (sts_regional === 'regional') { + /* STS regional endpoints can be derived from the region's name. + See: https://docs.aws.amazon.com/general/latest/gr/sts.html */ + const region = process.env['AWS_REGION']; + if (region) { + sts_endpoint = `https://sts.${region}.amazonaws.com`; + } else { + throw 'Missing required AWS_REGION env variable'; + } + } else { + // This is the default global endpoint + sts_endpoint = 'https://sts.amazonaws.com'; + } + } + + const token = fs.readFileSync(process.env['AWS_WEB_IDENTITY_TOKEN_FILE']); + + const params = `Version=2011-06-15&Action=AssumeRoleWithWebIdentity&RoleArn=${arn}&RoleSessionName=${name}&WebIdentityToken=${token}`; + + const response = await ngx.fetch(sts_endpoint + "?" + params, { + headers: { + "Accept": "application/json" + }, + method: 'GET', + }); + + const resp = await response.json(); + const creds = resp.AssumeRoleWithWebIdentityResponse.AssumeRoleWithWebIdentityResult.Credentials; + + return { + accessKeyId: creds.AccessKeyId, + secretAccessKey: creds.SecretAccessKey, + sessionToken: creds.SessionToken, + expiration: creds.Expiration, + }; +} + +/** + * Get the current timestamp. This timestamp will be used across functions in + * order for there to be no variations in signatures. + * + * @returns {Date} The current moment as a timestamp + */ +function Now() { + return NOW; +} + export default { + Now, + fetchCredentials, readCredentials, sessionToken, writeCredentials diff --git a/common/etc/nginx/include/awssig4.js b/common/etc/nginx/include/awssig4.js index 6206796c..7bf30e8f 100644 --- a/common/etc/nginx/include/awssig4.js +++ b/common/etc/nginx/include/awssig4.js @@ -14,22 +14,16 @@ * limitations under the License. */ -import utils from "./utils.js"; +import awscred from "./awscredentials.js"; +import utils from "./utils.js"; const mod_hmac = require('crypto'); -/** - * Constant checksum for an empty HTTP body. - * @type {string} - */ -const EMPTY_PAYLOAD_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; - /** * Constant defining the headers being signed. * @type {string} */ -const DEFAULT_SIGNED_HEADERS = 'host;x-amz-content-sha256;x-amz-date'; - +const DEFAULT_SIGNED_HEADERS = 'host;x-amz-date'; /** * Create HTTP Authorization header for authenticating with an AWS compatible @@ -48,13 +42,13 @@ const DEFAULT_SIGNED_HEADERS = 'host;x-amz-content-sha256;x-amz-date'; function signatureV4(r, timestamp, region, service, uri, queryParams, host, credentials) { const eightDigitDate = utils.getEightDigitDate(timestamp); const amzDatetime = utils.getAmzDatetime(timestamp, eightDigitDate); - const canonicalRequest = _buildCanonicalRequest( + const canonicalRequest = _buildCanonicalRequest(r, r.method, uri, queryParams, host, amzDatetime, credentials.sessionToken); const signature = _buildSignatureV4(r, amzDatetime, eightDigitDate, credentials, region, service, canonicalRequest); const authHeader = 'AWS4-HMAC-SHA256 Credential=' .concat(credentials.accessKeyId, '/', eightDigitDate, '/', region, '/', service, '/aws4_request,', - 'SignedHeaders=', _signedHeaders(credentials.sessionToken), ',Signature=', signature); + 'SignedHeaders=', _signedHeaders(r, credentials.sessionToken), ',Signature=', signature); utils.debug_log(r, 'AWS v4 Auth header: [' + authHeader + ']'); @@ -73,12 +67,13 @@ function signatureV4(r, timestamp, region, service, uri, queryParams, host, cred * @returns {string} string with concatenated request parameters * @private */ -function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, sessionToken) { +function _buildCanonicalRequest(r, + method, uri, queryParams, host, amzDatetime, sessionToken) { + const payloadHash = awsHeaderPayloadHash(r); let canonicalHeaders = 'host:' + host + '\n' + - 'x-amz-content-sha256:' + EMPTY_PAYLOAD_HASH + '\n' + - 'x-amz-date:' + amzDatetime + '\n'; + 'x-amz-date:' + amzDatetime + '\n'; - if (sessionToken) { + if (sessionToken && sessionToken.length > 0) { canonicalHeaders += 'x-amz-security-token:' + sessionToken + '\n' } @@ -86,9 +81,8 @@ function _buildCanonicalRequest(method, uri, queryParams, host, amzDatetime, ses canonicalRequest += uri + '\n'; canonicalRequest += queryParams + '\n'; canonicalRequest += canonicalHeaders + '\n'; - canonicalRequest += _signedHeaders(sessionToken) + '\n'; - canonicalRequest += EMPTY_PAYLOAD_HASH; - + canonicalRequest += _signedHeaders(r, sessionToken) + '\n'; + canonicalRequest += payloadHash; return canonicalRequest; } @@ -119,7 +113,7 @@ function _buildSignatureV4( const stringToSign = _buildStringToSign( amzDatetime, eightDigitDate, region, service, canonicalRequestHash); - utils.debug_log(r, 'AWS v4 Auth Signing String: [' + stringToSign + ']'); + utils.debug_log(r, 'AWS v4 Auth Signing String: [' + stringToSign + ']'); let kSigningHash; @@ -190,13 +184,14 @@ function _buildStringToSign(amzDatetime, eightDigitDate, region, service, canoni * Creates a string containing the headers that need to be signed as part of v4 * signature authentication. * + * @param r {Request} HTTP request object * @param sessionToken {string|undefined} AWS session token if present * @returns {string} semicolon delimited string of the headers needed for signing * @private */ -function _signedHeaders(sessionToken) { +function _signedHeaders(r, sessionToken) { let headers = DEFAULT_SIGNED_HEADERS; - if (sessionToken) { + if (sessionToken && sessionToken.length > 0) { headers += ';x-amz-security-token'; } return headers; @@ -250,8 +245,39 @@ function _splitCachedValues(cached) { return [eightDigitDate, kSigningHash] } +/** + * Outputs the timestamp used to sign the request, so that it can be added to + * the 'x-amz-date' header and sent by NGINX. The output format is + * ISO 8601: YYYYMMDD'T'HHMMSS'Z'. + * @see {@link https://docs.aws.amazon.com/general/latest/gr/sigv4-date-handling.html | Handling dates in Signature Version 4} + * + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @returns {string} ISO 8601 timestamp + */ +function awsHeaderDate(r) { + return utils.getAmzDatetime( + awscred.Now(), + utils.getEightDigitDate(awscred.Now()) + ); +} + +/** + * Return a payload hash in the header + * + * @param r {Request} HTTP request object + * @returns {string} payload hash + */ +function awsHeaderPayloadHash(r) { + const reqBody = r.variables.request_body ? r.variables.request_body: ''; + const payloadHash = mod_hmac.createHash('sha256', 'utf8') + .update(reqBody) + .digest('hex'); + return payloadHash; +} export default { + awsHeaderDate, + awsHeaderPayloadHash, signatureV4, // These functions do not need to be exposed, but they are exposed so that // unit tests can run against them. diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 6d694749..c5670654 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -27,7 +27,6 @@ _requireEnvVars('S3_REGION'); _requireEnvVars('AWS_SIGS_VERSION'); _requireEnvVars('S3_STYLE'); -const fs = require('fs'); /** * Flag indicating debug mode operation. If true, additional information @@ -48,33 +47,12 @@ const ADDITIONAL_HEADER_PREFIXES_TO_STRIP = utils.parseArray(process.env['HEADER */ const INDEX_PAGE = "index.html"; -/** - * The current moment as a timestamp. This timestamp will be used across - * functions in order for there to be no variations in signatures. - * @type {Date} - */ -const NOW = new Date(); - /** * Constant defining the service requests are being signed for. * @type {string} */ const SERVICE = 's3'; -/** - * Constant base URI to fetch credentials together with the credentials relative URI, see - * https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html for more details. - * @type {string} - */ -const ECS_CREDENTIAL_BASE_URI = 'http://169.254.170.2'; - -/** - * @type {string} - */ -const EC2_IMDS_TOKEN_ENDPOINT = 'http://169.254.169.254/latest/api/token'; - -const EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'; - /** * Transform the headers returned from S3 such that there isn't information * leakage about S3 and do other tasks needed for appropriate gateway output. @@ -139,20 +117,7 @@ function _isHeaderToBeStripped(headerName, additionalHeadersToStrip) { * @returns {string} RFC2616 timestamp */ function s3date(r) { - return NOW.toUTCString(); -} - -/** - * Outputs the timestamp used to sign the request, so that it can be added to - * the 'x-amz-date' header and sent by NGINX. The output format is - * ISO 8601: YYYYMMDD'T'HHMMSS'Z'. - * @see {@link https://docs.aws.amazon.com/general/latest/gr/sigv4-date-handling.html | Handling dates in Signature Version 4} - * - * @param r {Request} HTTP request object (not used, but required for NGINX configuration) - * @returns {string} ISO 8601 timestamp - */ -function awsHeaderDate(r) { - return utils.getAmzDatetime(NOW, utils.getEightDigitDate(NOW)); + return awscred.Now().toUTCString(); } /** @@ -181,7 +146,7 @@ function s3auth(r) { signature = awssig2.signatureV2(r, req.uri, req.httpDate, credentials); } else { let req = _s3ReqParamsForSigV4(r, bucket, server); - signature = awssig4.signatureV4(r, NOW, region, SERVICE, + signature = awssig4.signatureV4(r, awscred.Now(), region, SERVICE, req.uri, req.queryParams, req.host, credentials); } @@ -477,234 +442,7 @@ function _requireEnvVars(envVarName) { } } -/** - * Offset to the expiration of credentials, when they should be considered expired and refreshed. The maximum - * time here can be 5 minutes, the IMDS and ECS credentials endpoint will make sure that each returned set of credentials - * is valid for at least another 5 minutes. - * - * To make sure we always refresh the credentials instead of retrieving the same again, keep credentials until 4:30 minutes - * before they really expire. - * - * @type {number} - */ -const maxValidityOffsetMs = 4.5 * 60 * 1000; - -/** - * Get the credentials needed to create AWS signatures in order to authenticate - * to S3. If the gateway is being provided credentials via a instance profile - * credential as provided over the metadata endpoint, this function will: - * 1. Try to read the credentials from cache - * 2. Determine if the credentials are stale - * 3. If the cached credentials are missing or stale, it gets new credentials - * from the metadata endpoint. - * 4. If new credentials were pulled, it writes the credentials back to the - * cache. - * - * If the gateway is not using instance profile credentials, then this function - * quickly exits. - * - * @param r {Request} HTTP request object - * @returns {Promise} - */ -async function fetchCredentials(r) { - /* If we are not using an AWS instance profile to set our credentials we - exit quickly and don't write a credentials file. */ - if (utils.areAllEnvVarsSet(['S3_ACCESS_KEY_ID', 'S3_SECRET_KEY'])) { - r.return(200); - return; - } - - let current; - - try { - current = awscred.readCredentials(r); - } catch (e) { - utils.debug_log(r, `Could not read credentials: ${e}`); - r.return(500); - return; - } - - if (current) { - // If AWS returns a Unix timestamp it will be in seconds, but in Date constructor we should provide timestamp in milliseconds - // In some situations (including EC2 and Fargate) current.expiration will be an RFC 3339 string - see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials - const expireAt = typeof current.expiration == 'number' ? current.expiration * 1000 : current.expiration - const exp = new Date(expireAt).getTime() - maxValidityOffsetMs; - if (NOW.getTime() < exp) { - r.return(200); - return; - } - } - - let credentials; - - utils.debug_log(r, 'Cached credentials are expired or not present, requesting new ones'); - - if (utils.areAllEnvVarsSet('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')) { - const relative_uri = process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || ''; - const uri = ECS_CREDENTIAL_BASE_URI + relative_uri; - try { - credentials = await _fetchEcsRoleCredentials(uri); - } catch (e) { - utils.debug_log(r, 'Could not load ECS task role credentials: ' + JSON.stringify(e)); - r.return(500); - return; - } - } - else if (utils.areAllEnvVarsSet('AWS_WEB_IDENTITY_TOKEN_FILE')) { - try { - credentials = await _fetchWebIdentityCredentials(r) - } catch(e) { - utils.debug_log(r, 'Could not assume role using web identity: ' + JSON.stringify(e)); - r.return(500); - return; - } - } else { - try { - credentials = await _fetchEC2RoleCredentials(); - } catch (e) { - utils.debug_log(r, 'Could not load EC2 task role credentials: ' + JSON.stringify(e)); - r.return(500); - return; - } - } - try { - awscred.writeCredentials(r, credentials); - } catch (e) { - utils.debug_log(r, `Could not write credentials: ${e}`); - r.return(500); - return; - } - r.return(200); -} - -/** - * Get the credentials needed to generate AWS signatures from the ECS - * (Elastic Container Service) metadata endpoint. - * - * @param credentialsUri {string} endpoint to get credentials from - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchEcsRoleCredentials(credentialsUri) { - const resp = await ngx.fetch(credentialsUri); - if (!resp.ok) { - throw 'Credentials endpoint response was not ok.'; - } - const creds = await resp.json(); - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.Token, - expiration: creds.Expiration, - }; -} - -/** - * Get the credentials needed to generate AWS signatures from the EC2 - * metadata endpoint. - * - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchEC2RoleCredentials() { - const tokenResp = await ngx.fetch(EC2_IMDS_TOKEN_ENDPOINT, { - headers: { - 'x-aws-ec2-metadata-token-ttl-seconds': '21600', - }, - method: 'PUT', - }); - const token = await tokenResp.text(); - let resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT, { - headers: { - 'x-aws-ec2-metadata-token': token, - }, - }); - /* This _might_ get multiple possible roles in other scenarios, however, - EC2 supports attaching one role only.It should therefore be safe to take - the whole output, even given IMDS _might_ (?) be able to return multiple - roles. */ - const credName = await resp.text(); - if (credName === "") { - throw 'No credentials available for EC2 instance'; - } - resp = await ngx.fetch(EC2_IMDS_SECURITY_CREDENTIALS_ENDPOINT + credName, { - headers: { - 'x-aws-ec2-metadata-token': token, - }, - }); - const creds = await resp.json(); - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.Token, - expiration: creds.Expiration, - }; -} - -/** - * Get the credentials by assuming calling AssumeRoleWithWebIdentity with the environment variable - * values ROLE_ARN, AWS_WEB_IDENTITY_TOKEN_FILE and HOSTNAME - * - * @returns {Promise<{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}>} - * @private - */ -async function _fetchWebIdentityCredentials(r) { - const arn = process.env['AWS_ROLE_ARN']; - const name = process.env['HOSTNAME'] || 'nginx-s3-gateway'; - - let sts_endpoint = process.env['STS_ENDPOINT']; - if (!sts_endpoint) { - /* On EKS, the ServiceAccount can be annotated with - 'eks.amazonaws.com/sts-regional-endpoints' to control - the usage of regional endpoints. We are using the same standard - environment variable here as the AWS SDK. This is with the exception - of replacing the value `legacy` with `global` to match what EKS sets - the variable to. - See: https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html - See: https://docs.aws.amazon.com/eks/latest/userguide/configure-sts-endpoint.html */ - const sts_regional = process.env['AWS_STS_REGIONAL_ENDPOINTS'] || 'global'; - if (sts_regional === 'regional') { - /* STS regional endpoints can be derived from the region's name. - See: https://docs.aws.amazon.com/general/latest/gr/sts.html */ - const region = process.env['AWS_REGION']; - if (region) { - sts_endpoint = `https://sts.${region}.amazonaws.com`; - } else { - throw 'Missing required AWS_REGION env variable'; - } - } else { - // This is the default global endpoint - sts_endpoint = 'https://sts.amazonaws.com'; - } - } - - const token = fs.readFileSync(process.env['AWS_WEB_IDENTITY_TOKEN_FILE']); - - const params = `Version=2011-06-15&Action=AssumeRoleWithWebIdentity&RoleArn=${arn}&RoleSessionName=${name}&WebIdentityToken=${token}`; - - const response = await ngx.fetch(sts_endpoint + "?" + params, { - headers: { - "Accept": "application/json" - }, - method: 'GET', - }); - - const resp = await response.json(); - const creds = resp.AssumeRoleWithWebIdentityResponse.AssumeRoleWithWebIdentityResult.Credentials; - - return { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }; -} - export default { - awsHeaderDate, - fetchCredentials, s3date, s3auth, s3uri, diff --git a/common/etc/nginx/include/utils.js b/common/etc/nginx/include/utils.js index 9ab5f4fe..f5fa5627 100644 --- a/common/etc/nginx/include/utils.js +++ b/common/etc/nginx/include/utils.js @@ -19,7 +19,8 @@ * about signature generation will be logged. * @type {boolean} */ -const DEBUG = parseBoolean(process.env['S3_DEBUG']); +const DEBUG = parseBoolean(process.env['DEBUG']); + /** * Checks to see if all of the elements of the passed array are present as keys @@ -36,10 +37,8 @@ function areAllEnvVarsSet(envVars) { return false; } } - return true; } - return envVars in process.env; } @@ -148,12 +147,28 @@ function getEightDigitDate(timestamp) { padWithLeadingZeros(day,2)); } + +/** + * Checks to see if the given environment variable is present. If not, an error + * is thrown. + * @param envVarName {string} environment variable to check for + * @private + */ +function requireEnvVar(envVarName) { + const isSet = envVarName in process.env; + + if (!isSet) { + throw('Required environment variable ' + envVarName + ' is missing'); + } +} + export default { + areAllEnvVarsSet, debug_log, getAmzDatetime, getEightDigitDate, padWithLeadingZeros, parseArray, parseBoolean, - areAllEnvVarsSet + requireEnvVar } diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index da3776bf..b5cf4693 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -9,16 +9,16 @@ load_module modules/ngx_http_js_module.so; load_module modules/ngx_http_xslt_filter_module.so; # Preserve S3 environment variables for worker threads -env S3_ACCESS_KEY_ID; -env S3_SECRET_KEY; -env S3_SESSION_TOKEN; +env AWS_ACCESS_KEY_ID; +env AWS_SECRET_ACCESS_KEY; +env AWS_SESSION_TOKEN; env S3_BUCKET_NAME; env S3_SERVER; env S3_SERVER_PORT; env S3_SERVER_PROTO; env S3_REGION; env AWS_SIGS_VERSION; -env S3_DEBUG; +env DEBUG; env S3_STYLE; env ALLOW_DIRECTORY_LIST; env PROVIDE_INDEX_PAGE; diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index cf4cd605..b2dd54d2 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -1,6 +1,7 @@ js_import /etc/nginx/include/awscredentials.js; js_import /etc/nginx/include/s3gateway.js; + # We include only the variables needed for the authentication signatures that # we plan to use. include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_js_vars.conf; @@ -88,7 +89,7 @@ server { location /aws/credentials/retrieve { internal; - js_content s3gateway.fetchCredentials; + js_content awscredentials.fetchCredentials; include /etc/nginx/conf.d/gateway/js_fetch_trusted_certificate.conf; } diff --git a/common/etc/nginx/templates/gateway/v4_headers.conf.template b/common/etc/nginx/templates/gateway/v4_headers.conf.template index e6ca2daf..0227ed63 100644 --- a/common/etc/nginx/templates/gateway/v4_headers.conf.template +++ b/common/etc/nginx/templates/gateway/v4_headers.conf.template @@ -4,4 +4,4 @@ proxy_set_header x-amz-date $awsDate; # All HTTP bodies are empty because we are only doing GET/HEAD requests, # so we can hardcode the body checksum. -proxy_set_header x-amz-content-sha256 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; +proxy_set_header x-amz-content-sha256 $awsPayloadHash; diff --git a/common/etc/nginx/templates/gateway/v4_js_vars.conf.template b/common/etc/nginx/templates/gateway/v4_js_vars.conf.template index b7baeee4..3842a282 100644 --- a/common/etc/nginx/templates/gateway/v4_js_vars.conf.template +++ b/common/etc/nginx/templates/gateway/v4_js_vars.conf.template @@ -1,4 +1,7 @@ # This header is needed when doing v4 signature authentication. It # specifies the timestamp in which the signature was generated and is used with # the x-amz-date header. -js_set $awsDate s3gateway.awsHeaderDate; +js_import /etc/nginx/include/awssig4.js; + +js_set $awsDate awssig4.awsHeaderDate; +js_set $awsPayloadHash awssig4.awsHeaderPayloadHash; diff --git a/deployments/ecs/cloudformation/s3gateway.yaml b/deployments/ecs/cloudformation/s3gateway.yaml index a64bb2d1..79ebd44f 100644 --- a/deployments/ecs/cloudformation/s3gateway.yaml +++ b/deployments/ecs/cloudformation/s3gateway.yaml @@ -239,7 +239,7 @@ Resources: - amazonaws.com - Name: S3_STYLE Value: default - - Name: S3_DEBUG + - Name: DEBUG Value: 'false' Image: ghcr.io/nginxinc/nginx-s3-gateway/nginx-oss-s3-gateway:latest-njs-oss LogConfiguration: diff --git a/docs/getting_started.md b/docs/getting_started.md index 6704afca..db36ad7f 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -17,16 +17,16 @@ running as a Container or as a Systemd service. | ------------------------------------- | --------- | ---------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ALLOW_DIRECTORY_LIST` | Yes | `true`, `false` | `false` | Flag enabling directory listing | | `AWS_SIGS_VERSION` | Yes | 2, 4 | | AWS Signatures API version | -| `S3_ACCESS_KEY_ID` | Yes | | | Access key | -| `S3_SECRET_KEY` | Yes | | | Secret access key | -| `S3_SESSION_TOKEN` | No | | | Session token. | +| `AWS_ACCESS_KEY_ID` | Yes | | | Access key | +| `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | +| `AWS_SESSION_TOKEN` | No | | | Session token. | | `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | | `S3_REGION` | Yes | | | Region associated with API | | `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | | `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | | `S3_SERVER` | Yes | | | S3 host to connect to | | `S3_STYLE` | Yes | `virtual`, `path`, `default` | `default` | The S3 host/path method.
  • `virtual` is the method that that uses DNS-style bucket+hostname:port. This is the `default` value.
  • `path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services.

    See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information. | -| `S3_DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | +| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output | | `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. | | `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with | | `PROXY_CACHE_VALID_OK` | No | | | Sets caching time for response code 200 and 302 | @@ -40,7 +40,7 @@ running as a Container or as a Systemd service. If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), -you will need to omit the `S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` variables from +you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from the configuration. When running with Docker, the above environment variables can be set in a file @@ -51,7 +51,7 @@ file. There are few optional environment variables that can be used. -* `HOSTNAME` - (optional) The value will be used for Role Session Name. The default value is nginx-s3-gateway. +* `AWS_ROLE_SESSION_NAME` - (optional) The value will be used for Role Session Name. The default value is nginx-s3-gateway. * `STS_ENDPOINT` - (optional) Overrides the STS endpoint to be used in applicable setups. This is not required when running on EKS. See the EKS portion of the guide below for more details. * `AWS_STS_REGIONAL_ENDPOINTS` - (optional) Allows for a regional STS endpoint to be selected. When the regional model is selected then the STS endpoint generated will @@ -213,8 +213,8 @@ docker run --env-file ./settings --publish 80:80 --name nginx-plus-s3-gateway \ allow you to assign a role to a compute so that other AWS services can trust the instance without having to store authentication keys in the compute instance. This is useful for the gateway because it allows us to run the -gateway without storing an unchanging `S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and -`S3_SESSION_TOKEN` in a file on disk or in an easily read environment variable. +gateway without storing an unchanging `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and +`AWS_SESSION_TOKEN` in a file on disk or in an easily read environment variable. Instance profiles work by providing credentials to the instance via the [AWS Metadata API](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html). @@ -227,7 +227,7 @@ Following the [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/User we can create a IAM role and launch an instance associated with it. On that instance, if we run the gateway as a Systemd service there are no additional steps. We just run the install script without specifying the -`S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` environment variables. +`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` environment variables. However, if we want to run the gateway as a container instance on that EC2 instance, then we will need to run the following command using the AWS @@ -239,7 +239,7 @@ aws ec2 modify-instance-metadata-options --instance-id \ ``` After that has been run we can start the container normally and omit the -`S3_ACCESS_KEY_ID`, `S3_SECRET_KEY` and `S3_SESSION_TOKEN` environment variables. +`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` environment variables. ### Running in ECS with an IAM Policy diff --git a/settings.example b/settings.example index 9e55a4f8..4b8bbf32 100644 --- a/settings.example +++ b/settings.example @@ -1,13 +1,13 @@ S3_BUCKET_NAME=my-bucket -S3_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ -S3_SECRET_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -S3_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ +AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb S3_SERVER=s3-us-east-1.amazonaws.com S3_SERVER_PORT=443 S3_SERVER_PROTO=https S3_REGION=us-east-1 S3_STYLE=virtual -S3_DEBUG=false +DEBUG=false AWS_SIGS_VERSION=4 ALLOW_DIRECTORY_LIST=false PROVIDE_INDEX_PAGE=false diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index 7c4f4703..663f6470 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -40,11 +40,11 @@ elif curl --output /dev/null --silent --head --fail --connect-timeout 2 "http:// echo "Running inside an EC2 instance, using IMDS for credentials" uses_iam_creds=1 else - required+=("S3_ACCESS_KEY_ID" "S3_SECRET_KEY") + required+=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY") uses_iam_creds=0 fi -if [[ -v S3_SESSION_TOKEN ]]; then +if [[ -v AWS_SESSION_TOKEN ]]; then echo "S3 Session token present" fi @@ -78,7 +78,7 @@ echo "Installing using github '${branch}' branch" echo "S3 Backend Environment" -echo "Access Key ID: ${S3_ACCESS_KEY_ID}" +echo "Access Key ID: ${AWS_ACCESS_KEY_ID}" echo "Origin: ${S3_SERVER_PROTO}://${S3_BUCKET_NAME}.${S3_SERVER}:${S3_SERVER_PORT}" echo "Region: ${S3_REGION}" echo "Addressing Style: ${S3_STYLE}" @@ -148,7 +148,7 @@ S3_SERVER=${S3_SERVER} # The S3 host/path method - 'virtual', 'path' or 'default' S3_STYLE=${S3_STYLE} # Flag (true/false) enabling AWS signatures debug output (default: false) -S3_DEBUG=${S3_DEBUG} +DEBUG=${DEBUG} # Proxy caching time for response code 200 and 302 PROXY_CACHE_VALID_OK=${PROXY_CACHE_VALID_OK} # Proxy caching time for response code 404 @@ -187,14 +187,14 @@ EOF if [ $uses_iam_creds -eq 0 ]; then cat >> "/etc/nginx/environment" << EOF # AWS Access key -S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID} +AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # AWS Secret access key -S3_SECRET_KEY=${S3_SECRET_KEY} +AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} EOF - if [[ -v S3_SESSION_TOKEN ]]; then + if [[ -v AWS_SESSION_TOKEN ]]; then cat >> "/etc/nginx/environment" << EOF # AWS Session Token -S3_SESSION_TOKEN=${S3_SESSION_TOKEN} +AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} EOF fi fi @@ -293,12 +293,12 @@ EOF # to obtain S3 permissions. if [ $uses_iam_creds -eq 0 ]; then cat >> "/etc/nginx/environment" << EOF -env S3_ACCESS_KEY_ID; -env S3_SECRET_KEY; +env AWS_ACCESS_KEY_ID; +env AWS_SECRET_ACCESS_KEY; EOF - if [[ -v S3_SESSION_TOKEN ]]; then + if [[ -v AWS_SESSION_TOKEN ]]; then cat >> "/etc/nginx/environment" << EOF -env S3_SESSION_TOKEN; +env AWS_SESSION_TOKEN; EOF fi fi @@ -310,7 +310,7 @@ env S3_SERVER_PORT; env S3_SERVER_PROTO; env S3_REGION; env AWS_SIGS_VERSION; -env S3_DEBUG; +env DEBUG; env S3_STYLE; env ALLOW_DIRECTORY_LIST; diff --git a/test.sh b/test.sh index ca841707..2c2c4523 100755 --- a/test.sh +++ b/test.sh @@ -274,10 +274,10 @@ runUnitTestWithOutSessionToken() { --rm \ -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ - -e "S3_DEBUG=true" \ + -e "DEBUG=true" \ -e "S3_STYLE=virtual" \ - -e "S3_ACCESS_KEY_ID=unit_test" \ - -e "S3_SECRET_KEY=unit_test" \ + -e "AWS_ACCESS_KEY_ID=unit_test" \ + -e "AWS_SECRET_ACCESS_KEY=unit_test" \ -e "S3_BUCKET_NAME=unit_test" \ -e "S3_SERVER=unit_test" \ -e "S3_SERVER_PROTO=https" \ @@ -297,11 +297,11 @@ runUnitTestWithSessionToken() { --rm \ -v "$(pwd)/test/unit:/var/tmp" \ --workdir /var/tmp \ - -e "S3_DEBUG=true" \ + -e "DEBUG=true" \ -e "S3_STYLE=virtual" \ - -e "S3_ACCESS_KEY_ID=unit_test" \ - -e "S3_SECRET_KEY=unit_test" \ - -e "S3_SESSION_TOKEN=unit_test" \ + -e "AWS_ACCESS_KEY_ID=unit_test" \ + -e "AWS_SECRET_ACCESS_KEY=unit_test" \ + -e "AWS_SESSION_TOKEN=unit_test" \ -e "S3_BUCKET_NAME=unit_test" \ -e "S3_SERVER=unit_test" \ -e "S3_SERVER_PROTO=https" \ diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 8f088013..9106b81c 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -15,13 +15,13 @@ services: restart: "no" environment: S3_BUCKET_NAME: "bucket-1" - S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - S3_SECRET_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" + AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" S3_SERVER: "minio" S3_SERVER_PORT: "9000" S3_SERVER_PROTO: "http" S3_REGION: "us-east-1" - S3_DEBUG: "true" + DEBUG: "true" S3_STYLE: "virtual" ALLOW_DIRECTORY_LIST: PROVIDE_INDEX_PAGE: diff --git a/test/unit/awscredentials_test.js b/test/unit/awscredentials_test.js index 9a5a6d16..11eee36f 100644 --- a/test/unit/awscredentials_test.js +++ b/test/unit/awscredentials_test.js @@ -19,26 +19,28 @@ import awscred from "include/awscredentials.js"; import fs from "fs"; +globalThis.ngx = {}; + function testReadCredentialsWithAccessSecretKeyAndSessionTokenSet() { printHeader('testReadCredentialsWithAccessSecretKeyAndSessionTokenSet'); let r = {}; - process.env['S3_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY'; - process.env['S3_SECRET_KEY'] = 'SOME_SECRET_KEY'; - if ('S3_SESSION_TOKEN' in process.env) { - process.env['S3_SESSION_TOKEN'] = 'SOME_SESSION_TOKEN'; + process.env['AWS_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY'; + process.env['AWS_SECRET_ACCESS_KEY'] = 'SOME_SECRET_KEY'; + if ('AWS_SESSION_TOKEN' in process.env) { + process.env['AWS_SESSION_TOKEN'] = 'SOME_SESSION_TOKEN'; } try { var credentials = awscred.readCredentials(r); - if (credentials.accessKeyId !== process.env['S3_ACCESS_KEY_ID']) { + if (credentials.accessKeyId !== process.env['AWS_ACCESS_KEY_ID']) { throw 'static credentials do not match returned value [accessKeyId]'; } - if (credentials.secretAccessKey !== process.env['S3_SECRET_KEY']) { + if (credentials.secretAccessKey !== process.env['AWS_SECRET_ACCESS_KEY']) { throw 'static credentials do not match returned value [secretAccessKey]'; } - if ('S3_SESSION_TOKEN' in process.env) { - if (credentials.sessionToken !== process.env['S3_SESSION_TOKEN']) { + if ('AWS_SESSION_TOKEN' in process.env) { + if (credentials.sessionToken !== process.env['AWS_SESSION_TOKEN']) { throw 'static credentials do not match returned value [sessionToken]'; } } else { @@ -51,10 +53,10 @@ function testReadCredentialsWithAccessSecretKeyAndSessionTokenSet() { } } finally { - delete process.env.S3_ACCESS_KEY_ID; - delete process.env.S3_SECRET_KEY; - if ('S3_SESSION_TOKEN' in process.env) { - delete process.env.S3_SESSION_TOKEN; + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + if ('AWS_SESSION_TOKEN' in process.env) { + delete process.env.AWS_SESSION_TOKEN; } } } @@ -67,7 +69,7 @@ function testReadCredentialsFromFilePath() { } }; - var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; + var originalCredentialPath = process.env['AWS_CREDENTIALS_TEMP_FILE']; var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`; @@ -76,7 +78,7 @@ function testReadCredentialsFromFilePath() { fs.writeFileSync(tempFile, testData); try { - process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; + process.env['AWS_CREDENTIALS_TEMP_FILE'] = tempFile; var credentials = awscred.readCredentials(r); var testDataAsJSON = JSON.parse(testData); if (credentials.accessKeyId !== testDataAsJSON.accessKeyId) { @@ -93,7 +95,7 @@ function testReadCredentialsFromFilePath() { } } finally { if (originalCredentialPath) { - process.env['S3_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; + process.env['AWS_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; } if (fs.statSync(tempFile, {throwIfNoEntry: false})) { fs.unlinkSync(tempFile); @@ -108,13 +110,13 @@ function testReadCredentialsFromNonexistentPath() { cache_instance_credentials_enabled: 0 } }; - var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; + var originalCredentialPath = process.env['AWS_CREDENTIALS_TEMP_FILE']; var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`; try { - process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; + process.env['AWS_CREDENTIALS_TEMP_FILE'] = tempFile; var credentials = awscred.readCredentials(r); if (credentials !== undefined) { throw 'Credentials returned when no credentials file should be present'; @@ -122,7 +124,7 @@ function testReadCredentialsFromNonexistentPath() { } finally { if (originalCredentialPath) { - process.env['S3_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; + process.env['AWS_CREDENTIALS_TEMP_FILE'] = originalCredentialPath; } if (fs.statSync(tempFile, {throwIfNoEntry: false})) { fs.unlinkSync(tempFile); @@ -133,16 +135,16 @@ function testReadCredentialsFromNonexistentPath() { function testReadAndWriteCredentialsFromKeyValStore() { printHeader('testReadAndWriteCredentialsFromKeyValStore'); - let accessKeyId = process.env['S3_ACCESS_KEY_ID']; - let secretKey = process.env['S3_SECRET_KEY']; + let accessKeyId = process.env['AWS_ACCESS_KEY_ID']; + let secretKey = process.env['AWS_SECRET_ACCESS_KEY']; let sessionToken = null; - if ('S3_SESSION_TOKEN' in process.env) { - sessionToken = process.env['S3_SESSION_TOKEN']; + if ('AWS_SESSION_TOKEN' in process.env) { + sessionToken = process.env['AWS_SESSION_TOKEN']; } - delete process.env.S3_ACCESS_KEY_ID; - delete process.env.S3_SECRET_KEY; - if ('S3_SESSION_TOKEN' in process.env) { - delete process.env.S3_SESSION_TOKEN + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + if ('AWS_SESSION_TOKEN' in process.env) { + delete process.env.AWS_SESSION_TOKEN } try { let r = { @@ -168,15 +170,143 @@ function testReadAndWriteCredentialsFromKeyValStore() { throw 'Credentials do not match expected value'; } } finally { - process.env['S3_ACCESS_KEY_ID'] = accessKeyId; - process.env['S3_SECRET_KEY'] = secretKey; - if ('S3_SESSION_TOKEN' in process.env) { - process.env['S3_SESSION_TOKEN'] = sessionToken + process.env['AWS_ACCESS_KEY_ID'] = accessKeyId; + process.env['AWS_SECRET_ACCESS_KEY'] = secretKey; + if ('AWS_SESSION_TOKEN' in process.env) { + process.env['AWS_SESSION_TOKEN'] = sessionToken + } + } +} + +async function testEcsCredentialRetrieval() { + printHeader('testEcsCredentialRetrieval'); + if ('AWS_ACCESS_KEY_ID' in process.env) { + delete process.env['AWS_ACCESS_KEY_ID']; + } + process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = '/example'; + globalThis.ngx.fetch = function (url) { + console.log(' fetching mock credentials'); + globalThis.recordedUrl = url; + + return Promise.resolve({ + ok: true, + json: function () { + return Promise.resolve({ + AccessKeyId: 'AN_ACCESS_KEY_ID', + Expiration: '2017-05-17T15:09:54Z', + RoleArn: 'TASK_ROLE_ARN', + SecretAccessKey: 'A_SECRET_ACCESS_KEY', + Token: 'A_SECURITY_TOKEN', + }); + } + }); + }; + var r = { + "headersOut" : { + "Accept-Ranges": "bytes", + "Content-Length": 42, + "Content-Security-Policy": "block-all-mixed-content", + "Content-Type": "text/plain", + "X-Amz-Bucket-Region": "us-east-1", + "X-Amz-Request-Id": "166539E18A46500A", + "X-Xss-Protection": "1; mode=block" + }, + log: function(msg) { + console.log(msg); + }, + return: function(code) { + if (code !== 200) { + throw 'Expected 200 status code, got: ' + code; + } + }, + }; + + await awscred.fetchCredentials(r); + + if (globalThis.recordedUrl !== 'http://169.254.170.2/example') { + throw `No or wrong ECS credentials fetch URL recorded: ${globalThis.recordedUrl}`; + } +} + +async function testEc2CredentialRetrieval() { + printHeader('testEc2CredentialRetrieval'); + if ('AWS_ACCESS_KEY_ID' in process.env) { + delete process.env['AWS_ACCESS_KEY_ID']; + } + if ('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI' in process.env) { + delete process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']; + } + globalThis.ngx.fetch = function (url, options) { + if (url === 'http://169.254.169.254/latest/api/token' && options && options.method === 'PUT') { + return Promise.resolve({ + ok: true, + text: function () { + return Promise.resolve('A_TOKEN'); + }, + }); + } else if (url === 'http://169.254.169.254/latest/meta-data/iam/security-credentials/') { + if (options && options.headers && options.headers['x-aws-ec2-metadata-token'] === 'A_TOKEN') { + return Promise.resolve({ + ok: true, + text: function () { + return Promise.resolve('A_ROLE_NAME'); + }, + }); + } else { + throw 'Invalid token passed: ' + options.headers['x-aws-ec2-metadata-token']; + } + } else if (url === 'http://169.254.169.254/latest/meta-data/iam/security-credentials/A_ROLE_NAME') { + if (options && options.headers && options.headers['x-aws-ec2-metadata-token'] === 'A_TOKEN') { + return Promise.resolve({ + ok: true, + json: function () { + globalThis.credentialsIssued = true; + return Promise.resolve({ + AccessKeyId: 'AN_ACCESS_KEY_ID', + Expiration: '2017-05-17T15:09:54Z', + RoleArn: 'TASK_ROLE_ARN', + SecretAccessKey: 'A_SECRET_ACCESS_KEY', + Token: 'A_SECURITY_TOKEN', + }); + }, + }); + } else { + throw 'Invalid token passed: ' + options.headers['x-aws-ec2-metadata-token']; + } + } else { + throw 'Invalid request URL: ' + url; } + }; + var r = { + "headersOut" : { + "Accept-Ranges": "bytes", + "Content-Length": 42, + "Content-Security-Policy": "block-all-mixed-content", + "Content-Type": "text/plain", + "X-Amz-Bucket-Region": "us-east-1", + "X-Amz-Request-Id": "166539E18A46500A", + "X-Xss-Protection": "1; mode=block" + }, + log: function(msg) { + console.log(msg); + }, + return: function(code) { + if (code !== 200) { + throw 'Expected 200 status code, got: ' + code; + } + }, + }; + + await awscred.fetchCredentials(r); + + if (!globalThis.credentialsIssued) { + throw 'Did not reach the point where EC2 credentials were issues.'; } } async function test() { + await testEc2CredentialRetrieval(); + await testEcsCredentialRetrieval(); testReadCredentialsWithAccessSecretKeyAndSessionTokenSet(); testReadCredentialsFromFilePath(); testReadCredentialsFromNonexistentPath(); diff --git a/test/unit/awssig2_test.js b/test/unit/awssig2_test.js index 67b69080..6314375e 100644 --- a/test/unit/awssig2_test.js +++ b/test/unit/awssig2_test.js @@ -17,7 +17,6 @@ */ import awssig2 from "include/awssig2.js"; -import s3gateway from "include/s3gateway.js"; function _runSignatureV2(r) { @@ -32,12 +31,10 @@ function _runSignatureV2(r) { accessKeyId:accessKey, secretAccessKey: secret, sessionToken: null }; - // TODO: Generate request parameters without using s3gateway to only test - // awssig2.js for the purpose of common library. const httpDate = timestamp.toUTCString(); const expected = 'AWS test-access-key-1:VviSS4cFhUC6eoB4CYqtRawzDrc='; - let req = s3gateway._s3ReqParamsForSigV2(r, bucket); - let signature = awssig2.signatureV2(r, req.uri, httpDate, creds); + const req_uri = '/'.concat(bucket, r.variables.uri_path); + let signature = awssig2.signatureV2(r, req_uri, httpDate, creds); if (signature !== expected) { throw 'V2 signature hash was not created correctly.\n' + diff --git a/test/unit/awssig4_test.js b/test/unit/awssig4_test.js index 0c1677a1..351aed4c 100644 --- a/test/unit/awssig4_test.js +++ b/test/unit/awssig4_test.js @@ -17,7 +17,6 @@ */ import awssig4 from "include/awssig4.js"; -import s3gateway from "include/s3gateway.js"; import utils from "include/utils.js"; @@ -67,15 +66,17 @@ function _runSignatureV4(r) { var service = 's3'; var server = 's3-us-west-2.amazonaws.com'; - // TODO: Generate request parameters without using s3gateway to only test - // awssig4.js for the purpose of common library. - let req = s3gateway._s3ReqParamsForSigV4(r, bucket, server); - const canonicalRequest = awssig4._buildCanonicalRequest( + const req = { + uri : r.variables.uri_path, + queryParams : '', + host: bucket.concat('.', server) + } + const canonicalRequest = awssig4._buildCanonicalRequest(r, r.method, req.uri, req.queryParams, req.host, amzDatetime, creds.sessionToken); - var expected = 'cf4dd9e1d28c74e2284f938011efc8230d0c20704f56f67e4a3bfc2212026bec'; - var signature = awssig4._buildSignatureV4( - r, amzDatetime, eightDigitDate, creds, region, service, canonicalRequest); + var expected = '600721cacc21e3de14416de7517868381831f4709e5c5663bbf2b738e4d5abe4'; + var signature = awssig4._buildSignatureV4(r, + amzDatetime, eightDigitDate, creds, region, service, canonicalRequest); if (signature !== expected) { throw 'V4 signature hash was not created correctly.\n' + @@ -109,6 +110,7 @@ function testSignatureV4() { "foo" : "bar" }, "variables" : { + "request_body": "", "uri_path": "/a/c/ramen.jpg" }, "status" : 0 @@ -143,6 +145,7 @@ function testSignatureV4Cache() { }, "variables": { "cache_signing_key_enabled": 1, + "request_body": "", "uri_path": "/a/c/ramen.jpg" }, "status" : 0 diff --git a/test/unit/s3gateway_test.js b/test/unit/s3gateway_test.js index 2cd19154..bb23fa15 100755 --- a/test/unit/s3gateway_test.js +++ b/test/unit/s3gateway_test.js @@ -16,6 +16,7 @@ * limitations under the License. */ +import awscred from "include/awscredentials.js" import awssig4 from "include/awssig4.js"; import s3gateway from "include/s3gateway.js"; @@ -204,7 +205,7 @@ function testEscapeURIPathPreservesDoubleSlashes() { async function testEcsCredentialRetrieval() { printHeader('testEcsCredentialRetrieval'); - delete process.env['S3_ACCESS_KEY_ID']; + delete process.env['AWS_ACCESS_KEY_ID']; process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = '/example'; globalThis.ngx.fetch = function (url) { console.log(' fetching mock credentials'); @@ -243,7 +244,7 @@ async function testEcsCredentialRetrieval() { }, }; - await s3gateway.fetchCredentials(r); + await awscred.fetchCredentials(r); if (globalThis.recordedUrl !== 'http://169.254.170.2/example') { throw `No or wrong ECS credentials fetch URL recorded: ${globalThis.recordedUrl}`; @@ -252,7 +253,7 @@ async function testEcsCredentialRetrieval() { async function testEc2CredentialRetrieval() { printHeader('testEc2CredentialRetrieval'); - delete process.env['S3_ACCESS_KEY_ID']; + delete process.env['AWS_ACCESS_KEY_ID']; delete process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']; globalThis.ngx.fetch = function (url, options) { if (url === 'http://169.254.169.254/latest/api/token' && options && options.method === 'PUT') { @@ -315,7 +316,7 @@ async function testEc2CredentialRetrieval() { }, }; - await s3gateway.fetchCredentials(r); + await awscred.fetchCredentials(r); if (!globalThis.credentialsIssued) { throw 'Did not reach the point where EC2 credentials were issues.'; diff --git a/test/unit/utils_test.js b/test/unit/utils_test.js index 709f7149..addbea2b 100644 --- a/test/unit/utils_test.js +++ b/test/unit/utils_test.js @@ -181,6 +181,7 @@ function testAreAllEnvVarsSet() { testAreAllEnvVarsSetStringArrayWithSomeSet(); } + async function test() { testAmzDatetime(); testEightDigitDate(); @@ -188,7 +189,7 @@ async function test() { testParseArray(); testAreAllEnvVarsSet(); } - + function printHeader(testName) { console.log(`\n## ${testName}`); }