diff --git a/constants.js b/constants.js index 3b13af2f8e..9ef7dd39c4 100644 --- a/constants.js +++ b/constants.js @@ -111,6 +111,9 @@ const constants = { mpuMDStoredExternallyBackend: { aws_s3: true }, /* eslint-enable camelcase */ mpuMDStoredOnS3Backend: { azure: true }, + azureAccountNameRegex: /^[a-z0-9]{3,24}$/, + base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + + '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), }; module.exports = constants; diff --git a/lib/Config.js b/lib/Config.js index 1b7f61a33d..331ce35b39 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -5,9 +5,11 @@ const path = require('path'); const uuid = require('node-uuid'); +const { isValidBucketName } = require('arsenal').s3routes.routesUtils; const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig; const { buildAuthDataAccount } = require('./auth/in_memory/builder'); const externalBackends = require('../constants').externalBackends; +const { azureAccountNameRegex, base64Regex } = require('../constants'); // whitelist IP, CIDR for health checks const defaultHealthChecks = { allowFrom: ['127.0.0.1/8', '::1'] }; @@ -51,6 +53,46 @@ function restEndpointsAssert(restEndpoints, locationConstraints) { 'bad config: rest endpoint target not in locationConstraints'); } +function azureLocationConstraintAssert(location, locationObj) { + const { + azureStorageEndpoint, + azureStorageAccountName, + azureStorageAccessKey, + azureContainerName, + } = locationObj.details; + const storageEndpointFromEnv = + process.env[`${location}_AZURE_STORAGE_ENDPOINT`]; + const storageAccountNameFromEnv = + process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`]; + const storageAccessKeyFromEnv = + process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`]; + const locationParams = { + azureStorageEndpoint: storageEndpointFromEnv || azureStorageEndpoint, + azureStorageAccountName: + storageAccountNameFromEnv || azureStorageAccountName, + azureStorageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey, + azureContainerName, + }; + Object.keys(locationParams).forEach(param => { + const value = locationParams[param]; + assert.notEqual(value, undefined, + `bad location constraint: "${location}" ${param} ` + + 'must be set in locationConfig or environment variable'); + assert.strictEqual(typeof value, 'string', + `bad location constraint: "${location}" ${param} ` + + `"${value}" must be a string`); + }); + assert(azureAccountNameRegex.test(locationParams.azureStorageAccountName), + `bad location constraint: "${location}" azureStorageAccountName ` + + `"${locationParams.storageAccountName}" is an invalid value`); + assert(base64Regex.test(locationParams.azureStorageAccessKey), + `bad location constraint: "${location}" ` + + 'azureStorageAccessKey is not a valid base64 string'); + assert(isValidBucketName(azureContainerName, []), + `bad location constraint: "${location}" ` + + 'azureContainerName is an invalid container name'); +} + function locationConstraintAssert(locationConstraints) { const supportedBackends = ['mem', 'file', 'scality'].concat(Object.keys(externalBackends)); @@ -103,6 +145,9 @@ function locationConstraintAssert(locationConstraints) { assert(typeof details.credentials.secretKey === 'string', 'bad config: credentials must include secretKey as string'); } + if (locationConstraints[l].type === 'azure') { + azureLocationConstraintAssert(l, locationConstraints[l]); + } }); assert(Object.keys(locationConstraints) .includes('us-east-1'), 'bad locationConfig: must ' + @@ -752,19 +797,46 @@ class Config extends EventEmitter { } getAzureEndpoint(locationConstraint) { - return process.env[`${locationConstraint}_AZURE_BLOB_ENDPOINT`] || - this.locationConstraints[locationConstraint].details.azureBlobEndpoint; + let azureStorageEndpoint = + process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] || + this.locationConstraints[locationConstraint] + .details.azureStorageEndpoint; + if (!azureStorageEndpoint.endsWith('/')) { + // append the trailing slash + azureStorageEndpoint = `${azureStorageEndpoint}/`; + } + return azureStorageEndpoint; } - getAzureBlobSas(locationConstraint) { - return process.env[`${locationConstraint}_AZURE_BLOB_SAS`] || - this.locationConstraints[locationConstraint].details.azureBlobSAS; + getAzureStorageAccountName(locationConstraint) { + const { azureStorageAccountName } = + this.locationConstraints[locationConstraint].details; + const storageAccountNameFromEnv = + process.env[`${locationConstraint}_AZURE_STORAGE_ACCOUNT_NAME`]; + return storageAccountNameFromEnv || azureStorageAccountName; + } + + getAzureStorageCredentials(locationConstraint) { + const { azureStorageAccessKey } = + this.locationConstraints[locationConstraint].details; + const storageAccessKeyFromEnv = + process.env[`${locationConstraint}_AZURE_STORAGE_ACCESS_KEY`]; + return { + storageAccountName: + this.getAzureStorageAccountName(locationConstraint), + storageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey, + }; } isSameAzureAccount(locationConstraintSrc, locationConstraintDest) { - return locationConstraintDest ? - this.getAzureEndpoint(locationConstraintSrc) === - this.getAzureEndpoint(locationConstraintDest) : true; + if (!locationConstraintDest) { + return true; + } + const azureSrcAccount = + this.getAzureStorageAccountName(locationConstraintSrc); + const azureDestAccount = + this.getAzureStorageAccountName(locationConstraintDest); + return azureSrcAccount === azureDestAccount; } isAWSServerSideEncrytion(locationConstraint) { diff --git a/lib/data/external/AzureClient.js b/lib/data/external/AzureClient.js index 91f2f6947a..0b20d88c01 100644 --- a/lib/data/external/AzureClient.js +++ b/lib/data/external/AzureClient.js @@ -11,11 +11,13 @@ const azureMpuUtils = s3middleware.azureHelper.mpuUtils; class AzureClient { constructor(config) { - this._azureBlobEndpoint = config.azureBlobEndpoint; - this._azureBlobSAS = config.azureBlobSAS; + this._azureStorageEndpoint = config.azureStorageEndpoint; + this._azureStorageCredentials = config.azureStorageCredentials; this._azureContainerName = config.azureContainerName; - this._client = azure.createBlobServiceWithSas( - this._azureBlobEndpoint, this._azureBlobSAS); + this._client = azure.createBlobService( + this._azureStorageCredentials.storageAccountName, + this._azureStorageCredentials.storageAccessKey, + this._azureStorageEndpoint); this._dataStoreName = config.dataStoreName; this._bucketMatch = config.bucketMatch; } @@ -226,7 +228,8 @@ class AzureClient { return callback(null, azureResp); } azureResp[location] = { - message: 'Congrats! You own the azure container', + message: + 'Congrats! You can access the Azure storage account', }; return callback(null, azureResp); }], null, callback); @@ -361,8 +364,9 @@ class AzureClient { .details.azureContainerName; this._errorWrapper('copyObject', 'startCopyBlob', - [`${this._azureBlobEndpoint}/${sourceContainerName}/${sourceKey}` + - `?${this._azureBlobSAS}`, this._azureContainerName, destAzureKey, + [`${this._azureStorageEndpoint}` + + `${sourceContainerName}/${sourceKey}`, + this._azureContainerName, destAzureKey, (err, res) => { if (err) { if (err.code === 'CannotVerifyCopySource') { diff --git a/lib/data/locationConstraintParser.js b/lib/data/locationConstraintParser.js index 5d12fd235e..795cce0c01 100644 --- a/lib/data/locationConstraintParser.js +++ b/lib/data/locationConstraintParser.js @@ -74,11 +74,12 @@ function parseLC() { clients[location].clientType = 'aws_s3'; } if (locationObj.type === 'azure') { - const azureBlobEndpoint = config.getAzureEndpoint(location); - const azureBlobSAS = config.getAzureBlobSas(location); + const azureStorageEndpoint = config.getAzureEndpoint(location); + const azureStorageCredentials = + config.getAzureStorageCredentials(location); clients[location] = new AzureClient({ - azureBlobEndpoint, - azureBlobSAS, + azureStorageEndpoint, + azureStorageCredentials, azureContainerName: locationObj.details.azureContainerName, bucketMatch: locationObj.details.bucketMatch, dataStoreName: location, diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js index 03fd2e48a1..8129740e4b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js @@ -61,44 +61,35 @@ const utils = { utils.uniqName = name => `${name}${new Date().getTime()}`; utils.getAzureClient = () => { - let isTestingAzure; - let azureBlobEndpoint; - let azureBlobSAS; - let azureClient; - if (process.env[`${azureLocation}_AZURE_BLOB_ENDPOINT`]) { - isTestingAzure = true; - azureBlobEndpoint = process.env[`${azureLocation}_AZURE_BLOB_ENDPOINT`]; - } else if (config.locationConstraints[azureLocation] && - config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details.azureBlobEndpoint) { - isTestingAzure = true; - azureBlobEndpoint = - config.locationConstraints[azureLocation].details.azureBlobEndpoint; - } else { - isTestingAzure = false; - } + const params = {}; + const envMap = { + azureStorageEndpoint: 'AZURE_STORAGE_ENDPOINT', + azureStorageAccountName: 'AZURE_STORAGE_ACCOUNT_NAME', + azureStorageAccessKey: 'AZURE_STORAGE_ACCESS_KEY', + }; - if (isTestingAzure) { - if (process.env[`${azureLocation}_AZURE_BLOB_SAS`]) { - azureBlobSAS = process.env[`${azureLocation}_AZURE_BLOB_SAS`]; - isTestingAzure = true; - } else if (config.locationConstraints[azureLocation] && + const isTestingAzure = Object.keys(envMap).every(key => { + const envVariable = process.env[`${azureLocation}_${envMap[key]}`]; + if (envVariable) { + params[key] = envVariable; + return true; + } + if (config.locationConstraints[azureLocation] && config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details.azureBlobSAS - ) { - azureBlobSAS = config.locationConstraints[azureLocation].details - .azureBlobSAS; - isTestingAzure = true; - } else { - isTestingAzure = false; + config.locationConstraints[azureLocation].details[key]) { + params[key] = + config.locationConstraints[azureLocation].details[key]; + return true; } - } + return false; + }); - if (isTestingAzure) { - azureClient = azure.createBlobServiceWithSas(azureBlobEndpoint, - azureBlobSAS); + if (!isTestingAzure) { + return undefined; } - return azureClient; + + return azure.createBlobService(params.azureStorageAccountName, + params.azureStorageAccessKey, params.azureStorageEndpoint); }; utils.getAzureContainerName = () => { diff --git a/tests/locationConfigTests.json b/tests/locationConfigTests.json index eba01876dc..6485048963 100644 --- a/tests/locationConfigTests.json +++ b/tests/locationConfigTests.json @@ -75,30 +75,33 @@ "type": "azure", "legacyAwsBehavior": true, "details": { - "azureBlobEndpoint": "https://mystique.blob.core.fake.net", - "bucketMatch": true, - "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D", - "azureContainerName": "s3test" + "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/", + "azureStorageAccountName": "fakeaccountname", + "azureStorageAccessKey": "Fake00Key001", + "bucketMatch": true, + "azureContainerName": "s3test" } }, "azuretest2": { "type": "azure", "legacyAwsBehavior": true, "details": { - "azureBlobEndpoint": "https://mystique2.blob.core.fake.net", - "bucketMatch": true, - "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D", - "azureContainerName": "s3test2" + "azureStorageEndpoint": "https://fakeaccountname2.blob.core.fake.net/", + "azureStorageAccountName": "fakeaccountname2", + "azureStorageAccessKey": "Fake00Key002", + "bucketMatch": true, + "azureContainerName": "s3test2" } }, "azuretestmismatch": { "type": "azure", "legacyAwsBehavior": true, "details": { - "azureBlobEndpoint": "https://mystique.blob.core.fake.net", - "bucketMatch": false, - "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D", - "azureContainerName": "s3test" + "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/", + "azureStorageAccountName": "fakeaccountname", + "azureStorageAccessKey": "Fake00Key001", + "bucketMatch": false, + "azureContainerName": "s3test" } } } diff --git a/tests/unit/testConfigs/locConstraintAssert.js b/tests/unit/testConfigs/locConstraintAssert.js index 1ade534466..a190ac5cb5 100644 --- a/tests/unit/testConfigs/locConstraintAssert.js +++ b/tests/unit/testConfigs/locConstraintAssert.js @@ -13,6 +13,16 @@ class LocationConstraint { } } +function getAzureDetails(replaceParams) { + return Object.assign({ + azureStorageEndpoint: 'https://fakeaccountname.blob.core.fake.net/', + azureStorageAccountName: 'fakeaccountname', + azureStorageAccessKey: 'Fake00Key123', + bucketMatch: false, + azureContainerName: 'test', + }, replaceParams); +} + describe('locationConstraintAssert', () => { it('should throw error if locationConstraints is not an object', () => { assert.throws(() => { @@ -103,4 +113,115 @@ describe('locationConstraintAssert', () => { '/bad locationConfig: must ' + 'include us-east-1 as a locationConstraint/'); }); + it('should not throw error for a valid azure location constraint', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails()); + assert.doesNotThrow(() => { + locationConstraintAssert({ 'azurefaketest': locationConstraint, + 'us-east-1': usEast1 }); + }, + '/should not throw for a valid azure location constraint/'); + }); + it('should throw error if type is azure and azureContainerName is ' + + 'not specified', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureContainerName: undefined })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: ' + + '"azurefaketest" azureContainerName must be defined/'); + }); + it('should throw error if type is azure and azureContainerName is ' + + 'invalid value', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureContainerName: '.' })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureContainerName is an invalid container name/'); + }); + it('should throw error if type is azure and azureStorageEndpoint ' + + 'is not specified', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureStorageEndpoint: undefined })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageEndpoint must be set in locationConfig ' + + 'or environment variable/'); + }); + it('should throw error if type is azure and azureStorageAccountName ' + + 'is not specified', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureStorageAccountName: undefined })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccountName must be set in locationConfig ' + + 'or environment variable/'); + }); + it('should throw error if type is azure and azureStorageAccountName ' + + 'is invalid value', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureStorageAccountName: 'invalid!!!' })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccountName "invalid!!!" is an invalid value/'); + }); + it('should throw error if type is azure and azureStorageAccessKey ' + + 'is not specified', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureStorageAccessKey: undefined })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccessKey must be set in locationConfig ' + + 'or environment variable/'); + }); + it('should throw error if type is azure and azureStorageAccessKey ' + + 'is not a valid base64 string', () => { + const usEast1 = new LocationConstraint(); + const locationConstraint = new LocationConstraint('azure', true, + getAzureDetails({ azureStorageAccessKey: 'invalid!!!' })); + assert.throws(() => { + locationConstraintAssert({ + 'us-east-1': usEast1, + 'azurefaketest': locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccessKey is not a valid base64 string/'); + }); });