rf: azure location config, use account / key

scality · Oct 30, 2017 · 1ce30b4 · 1ce30b4
1 parent 7d230b5
commit 1ce30b4
Show file tree

Hide file tree

Showing 7 changed files with 259 additions and 64 deletions.
diff --git a/constants.js b/constants.js
@@ -111,6 +111,9 @@ const constants = {
     mpuMDStoredExternallyBackend: { aws_s3: true },
     /* eslint-enable camelcase */
     mpuMDStoredOnS3Backend: { azure: true },
+    azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
+    base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
+        '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
 };
 
 module.exports = constants;
diff --git a/lib/Config.js b/lib/Config.js
@@ -5,9 +5,11 @@ const path = require('path');
 
 const uuid = require('node-uuid');
 
+const { isValidBucketName } = require('arsenal').s3routes.routesUtils;
 const validateAuthConfig = require('arsenal').auth.inMemory.validateAuthConfig;
 const { buildAuthDataAccount } = require('./auth/in_memory/builder');
 const externalBackends = require('../constants').externalBackends;
+const { azureAccountNameRegex, base64Regex } = require('../constants');
 
 // whitelist IP, CIDR for health checks
 const defaultHealthChecks = { allowFrom: ['127.0.0.1/8', '::1'] };
@@ -51,6 +53,46 @@ function restEndpointsAssert(restEndpoints, locationConstraints) {
         'bad config: rest endpoint target not in locationConstraints');
 }
 
+function azureLocationConstraintAssert(location, locationObj) {
+    const {
+        azureStorageEndpoint,
+        azureStorageAccountName,
+        azureStorageAccessKey,
+        azureContainerName,
+    } = locationObj.details;
+    const storageEndpointFromEnv =
+        process.env[`${location}_AZURE_STORAGE_ENDPOINT`];
+    const storageAccountNameFromEnv =
+        process.env[`${location}_AZURE_STORAGE_ACCOUNT_NAME`];
+    const storageAccessKeyFromEnv =
+        process.env[`${location}_AZURE_STORAGE_ACCESS_KEY`];
+    const locationParams = {
+        azureStorageEndpoint: storageEndpointFromEnv || azureStorageEndpoint,
+        azureStorageAccountName:
+            storageAccountNameFromEnv || azureStorageAccountName,
+        azureStorageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
+        azureContainerName,
+    };
+    Object.keys(locationParams).forEach(param => {
+        const value = locationParams[param];
+        assert.notEqual(value, undefined,
+            `bad location constraint: "${location}" ${param} ` +
+            'must be set in locationConfig or environment variable');
+        assert.strictEqual(typeof value, 'string',
+            `bad location constraint: "${location}" ${param} ` +
+            `"${value}" must be a string`);
+    });
+    assert(azureAccountNameRegex.test(locationParams.azureStorageAccountName),
+        `bad location constraint: "${location}" azureStorageAccountName ` +
+        `"${locationParams.storageAccountName}" is an invalid value`);
+    assert(base64Regex.test(locationParams.azureStorageAccessKey),
+        `bad location constraint: "${location}" ` +
+        'azureStorageAccessKey is not a valid base64 string');
+    assert(isValidBucketName(azureContainerName, []),
+        `bad location constraint: "${location}" ` +
+        'azureContainerName is an invalid container name');
+}
+
 function locationConstraintAssert(locationConstraints) {
     const supportedBackends =
       ['mem', 'file', 'scality'].concat(Object.keys(externalBackends));
@@ -103,6 +145,9 @@ function locationConstraintAssert(locationConstraints) {
             assert(typeof details.credentials.secretKey === 'string',
                 'bad config: credentials must include secretKey as string');
         }
+        if (locationConstraints[l].type === 'azure') {
+            azureLocationConstraintAssert(l, locationConstraints[l]);
+        }
     });
     assert(Object.keys(locationConstraints)
         .includes('us-east-1'), 'bad locationConfig: must ' +
@@ -752,19 +797,46 @@ class Config extends EventEmitter {
     }
 
     getAzureEndpoint(locationConstraint) {
-        return process.env[`${locationConstraint}_AZURE_BLOB_ENDPOINT`] ||
-        this.locationConstraints[locationConstraint].details.azureBlobEndpoint;
+        let azureStorageEndpoint =
+        process.env[`${locationConstraint}_AZURE_STORAGE_ENDPOINT`] ||
+        this.locationConstraints[locationConstraint]
+            .details.azureStorageEndpoint;
+        if (!azureStorageEndpoint.endsWith('/')) {
+            // append the trailing slash
+            azureStorageEndpoint = `${azureStorageEndpoint}/`;
+        }
+        return azureStorageEndpoint;
     }
 
-    getAzureBlobSas(locationConstraint) {
-        return process.env[`${locationConstraint}_AZURE_BLOB_SAS`] ||
-        this.locationConstraints[locationConstraint].details.azureBlobSAS;
+    getAzureStorageAccountName(locationConstraint) {
+        const { azureStorageAccountName } =
+            this.locationConstraints[locationConstraint].details;
+        const storageAccountNameFromEnv =
+            process.env[`${locationConstraint}_AZURE_STORAGE_ACCOUNT_NAME`];
+        return storageAccountNameFromEnv || azureStorageAccountName;
+    }
+
+    getAzureStorageCredentials(locationConstraint) {
+        const { azureStorageAccessKey } =
+            this.locationConstraints[locationConstraint].details;
+        const storageAccessKeyFromEnv =
+            process.env[`${locationConstraint}_AZURE_STORAGE_ACCESS_KEY`];
+        return {
+            storageAccountName:
+                this.getAzureStorageAccountName(locationConstraint),
+            storageAccessKey: storageAccessKeyFromEnv || azureStorageAccessKey,
+        };
     }
 
     isSameAzureAccount(locationConstraintSrc, locationConstraintDest) {
-        return locationConstraintDest ?
-        this.getAzureEndpoint(locationConstraintSrc) ===
-        this.getAzureEndpoint(locationConstraintDest) : true;
+        if (!locationConstraintDest) {
+            return true;
+        }
+        const azureSrcAccount =
+            this.getAzureStorageAccountName(locationConstraintSrc);
+        const azureDestAccount =
+            this.getAzureStorageAccountName(locationConstraintDest);
+        return azureSrcAccount === azureDestAccount;
     }
 
     isAWSServerSideEncrytion(locationConstraint) {

diff --git a/lib/data/external/AzureClient.js b/lib/data/external/AzureClient.js
@@ -11,11 +11,13 @@ const azureMpuUtils = s3middleware.azureHelper.mpuUtils;
 
 class AzureClient {
     constructor(config) {
-        this._azureBlobEndpoint = config.azureBlobEndpoint;
-        this._azureBlobSAS = config.azureBlobSAS;
+        this._azureStorageEndpoint = config.azureStorageEndpoint;
+        this._azureStorageCredentials = config.azureStorageCredentials;
         this._azureContainerName = config.azureContainerName;
-        this._client = azure.createBlobServiceWithSas(
-          this._azureBlobEndpoint, this._azureBlobSAS);
+        this._client = azure.createBlobService(
+            this._azureStorageCredentials.storageAccountName,
+            this._azureStorageCredentials.storageAccessKey,
+            this._azureStorageEndpoint);
         this._dataStoreName = config.dataStoreName;
         this._bucketMatch = config.bucketMatch;
     }
@@ -226,7 +228,8 @@ class AzureClient {
                     return callback(null, azureResp);
                 }
                 azureResp[location] = {
-                    message: 'Congrats! You own the azure container',
+                    message:
+                    'Congrats! You can access the Azure storage account',
                 };
                 return callback(null, azureResp);
             }], null, callback);
@@ -361,8 +364,9 @@ class AzureClient {
         .details.azureContainerName;
 
         this._errorWrapper('copyObject', 'startCopyBlob',
-            [`${this._azureBlobEndpoint}/${sourceContainerName}/${sourceKey}` +
-            `?${this._azureBlobSAS}`, this._azureContainerName, destAzureKey,
+            [`${this._azureStorageEndpoint}` +
+                `${sourceContainerName}/${sourceKey}`,
+                this._azureContainerName, destAzureKey,
             (err, res) => {
                 if (err) {
                     if (err.code === 'CannotVerifyCopySource') {

diff --git a/lib/data/locationConstraintParser.js b/lib/data/locationConstraintParser.js
@@ -74,11 +74,12 @@ function parseLC() {
             clients[location].clientType = 'aws_s3';
         }
         if (locationObj.type === 'azure') {
-            const azureBlobEndpoint = config.getAzureEndpoint(location);
-            const azureBlobSAS = config.getAzureBlobSas(location);
+            const azureStorageEndpoint = config.getAzureEndpoint(location);
+            const azureStorageCredentials =
+                config.getAzureStorageCredentials(location);
             clients[location] = new AzureClient({
-                azureBlobEndpoint,
-                azureBlobSAS,
+                azureStorageEndpoint,
+                azureStorageCredentials,
                 azureContainerName: locationObj.details.azureContainerName,
                 bucketMatch: locationObj.details.bucketMatch,
                 dataStoreName: location,

diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js
@@ -61,44 +61,35 @@ const utils = {
 utils.uniqName = name => `${name}${new Date().getTime()}`;
 
 utils.getAzureClient = () => {
-    let isTestingAzure;
-    let azureBlobEndpoint;
-    let azureBlobSAS;
-    let azureClient;
-    if (process.env[`${azureLocation}_AZURE_BLOB_ENDPOINT`]) {
-        isTestingAzure = true;
-        azureBlobEndpoint = process.env[`${azureLocation}_AZURE_BLOB_ENDPOINT`];
-    } else if (config.locationConstraints[azureLocation] &&
-          config.locationConstraints[azureLocation].details &&
-          config.locationConstraints[azureLocation].details.azureBlobEndpoint) {
-        isTestingAzure = true;
-        azureBlobEndpoint =
-          config.locationConstraints[azureLocation].details.azureBlobEndpoint;
-    } else {
-        isTestingAzure = false;
-    }
+    const params = {};
+    const envMap = {
+        azureStorageEndpoint: 'AZURE_STORAGE_ENDPOINT',
+        azureStorageAccountName: 'AZURE_STORAGE_ACCOUNT_NAME',
+        azureStorageAccessKey: 'AZURE_STORAGE_ACCESS_KEY',
+    };
 
-    if (isTestingAzure) {
-        if (process.env[`${azureLocation}_AZURE_BLOB_SAS`]) {
-            azureBlobSAS = process.env[`${azureLocation}_AZURE_BLOB_SAS`];
-            isTestingAzure = true;
-        } else if (config.locationConstraints[azureLocation] &&
+    const isTestingAzure = Object.keys(envMap).every(key => {
+        const envVariable = process.env[`${azureLocation}_${envMap[key]}`];
+        if (envVariable) {
+            params[key] = envVariable;
+            return true;
+        }
+        if (config.locationConstraints[azureLocation] &&
             config.locationConstraints[azureLocation].details &&
-            config.locationConstraints[azureLocation].details.azureBlobSAS
-        ) {
-            azureBlobSAS = config.locationConstraints[azureLocation].details
-              .azureBlobSAS;
-            isTestingAzure = true;
-        } else {
-            isTestingAzure = false;
+            config.locationConstraints[azureLocation].details[key]) {
+            params[key] =
+                config.locationConstraints[azureLocation].details[key];
+            return true;
         }
-    }
+        return false;
+    });
 
-    if (isTestingAzure) {
-        azureClient = azure.createBlobServiceWithSas(azureBlobEndpoint,
-          azureBlobSAS);
+    if (!isTestingAzure) {
+        return undefined;
     }
-    return azureClient;
+
+    return azure.createBlobService(params.azureStorageAccountName,
+        params.azureStorageAccessKey, params.azureStorageEndpoint);
 };
 
 utils.getAzureContainerName = () => {

diff --git a/tests/locationConfigTests.json b/tests/locationConfigTests.json
@@ -75,30 +75,33 @@
         "type": "azure",
         "legacyAwsBehavior": true,
         "details": {
-          "azureBlobEndpoint": "https://mystique.blob.core.fake.net",
-          "bucketMatch": true,
-          "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D",
-          "azureContainerName": "s3test"
+            "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/",
+            "azureStorageAccountName": "fakeaccountname",
+            "azureStorageAccessKey": "Fake00Key001",
+            "bucketMatch": true,
+            "azureContainerName": "s3test"
         }
     },
     "azuretest2": {
         "type": "azure",
         "legacyAwsBehavior": true,
         "details": {
-          "azureBlobEndpoint": "https://mystique2.blob.core.fake.net",
-          "bucketMatch": true,
-          "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D",
-          "azureContainerName": "s3test2"
+            "azureStorageEndpoint": "https://fakeaccountname2.blob.core.fake.net/",
+            "azureStorageAccountName": "fakeaccountname2",
+            "azureStorageAccessKey": "Fake00Key002",
+            "bucketMatch": true,
+            "azureContainerName": "s3test2"
         }
     },
     "azuretestmismatch": {
         "type": "azure",
         "legacyAwsBehavior": true,
         "details": {
-          "azureBlobEndpoint": "https://mystique.blob.core.fake.net",
-          "bucketMatch": false,
-          "azureBlobSAS": "sv=2015-04-05&sr=b&si=tutorial-policy-635959936345100803&sig=9aCzs76n0E7y5BpEi2GvsSv433BZa22leDOZXX%2BXXIU%3D",
-          "azureContainerName": "s3test"
+            "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/",
+            "azureStorageAccountName": "fakeaccountname",
+            "azureStorageAccessKey": "Fake00Key001",
+            "bucketMatch": false,
+            "azureContainerName": "s3test"
         }
     }
 }