diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 55e8ef426..521813adc 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -17,7 +17,6 @@ const levenshtein = require('fastest-levenshtein'); const RowMode = require('./../constants/row_mode'); const DataTypes = require('./result/data_types'); const Logger = require('../logger'); -const LoggingUtil = require('../logger/logging_util'); const WAIT_FOR_BROWSER_ACTION_TIMEOUT = 120000; const DEFAULT_PARAMS = [ @@ -513,7 +512,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) { passcode = options.passcode; } - if (validateDefaultParameters) { for (const [key] of Object.entries(options)) { @@ -847,21 +845,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) { this.describeIdentityAttributes = function () { return `host: ${this.host}, account: ${this.account}, accessUrl: ${this.accessUrl}, ` + `user: ${this.username}, role: ${this.getRole()}, database: ${this.getDatabase()}, ` - + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + this.describeProxy(); - }; - - /** - * @returns {string} - */ - this.describeProxy = function () { - const proxy = this.getProxy(); - if (Util.exists(proxy)) { - return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, ` - + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, ` - + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`; - } else { - return 'proxy was not configured'; - } + + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + ProxyUtil.describeProxy(this.getProxy()); }; // save config options diff --git a/lib/file_transfer_agent/azure_util.js b/lib/file_transfer_agent/azure_util.js index 005c403bd..e4f76e61c 100644 --- a/lib/file_transfer_agent/azure_util.js +++ b/lib/file_transfer_agent/azure_util.js @@ -52,7 +52,6 @@ function AzureUtil(connectionConfig, azure, filestream) { Logger.getInstance().trace(`Initializing the proxy information for the Azure Client: ${ProxyUtil.describeProxy(proxy)}`); proxy = ProxyUtil.getAzureProxy(proxy); - Logger.getInstance().trace(connectionConfig.describe); } ProxyUtil.hideEnvironmentProxy(); const blobServiceClient = new AZURE.BlobServiceClient( diff --git a/lib/file_transfer_agent/gcs_util.js b/lib/file_transfer_agent/gcs_util.js index c565a204d..f38e11cbb 100644 --- a/lib/file_transfer_agent/gcs_util.js +++ b/lib/file_transfer_agent/gcs_util.js @@ -4,8 +4,12 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; +const getProxyAgent = require('../http/node').getProxyAgent; +const ProxyUtil = require('../proxy_util'); +const Util = require('../util'); const { shouldPerformGCPBucket, lstrip } = require('../util'); + const GCS_METADATA_PREFIX = 'x-goog-meta-'; const SFC_DIGEST = 'sfc-digest'; const MATDESC_KEY = 'matdesc'; @@ -19,7 +23,6 @@ const GCS_FILE_HEADER_ENCRYPTION_METADATA = 'gcs-file-header-encryption-metadata const HTTP_HEADER_CONTENT_ENCODING = 'Content-Encoding'; const resultStatus = require('./file_util').resultStatus; - const { Storage } = require('@google-cloud/storage'); const EXPIRED_TOKEN = 'ExpiredToken'; @@ -40,17 +43,18 @@ function GCSLocation(bucketName, path) { /** * Creates an GCS utility object. - * - * @param {module} httpclient - * @param {module} filestream + * @param {module} connectionConfig + * @param {module} httpClient + * @param {module} fileStream * * @returns {Object} * @constructor */ -function GCSUtil(httpclient, filestream) { - const axios = typeof httpclient !== 'undefined' ? httpclient : require('axios'); - const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); - +function GCSUtil(connectionConfig, httpClient, fileStream) { + let axios = httpClient; + const fs = typeof fileStream !== 'undefined' ? fileStream : require('fs'); + let isProxyEnabled = false; + /** * Retrieve the GCS token from the stage info metadata. * @@ -61,7 +65,15 @@ function GCSUtil(httpclient, filestream) { this.createClient = function (stageInfo) { const stageCredentials = stageInfo['creds']; const gcsToken = stageCredentials['GCS_ACCESS_TOKEN']; - + //TODO: SNOW-1789759 the value is hardcoded now, but it should be server driven + const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl; + let endPoint = null; + if (stageInfo['endPoint']) { + endPoint = stageInfo['endPoint']; + } else if (isRegionalUrlEnabled) { + endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`; + } + let client; if (gcsToken) { const interceptors = []; @@ -72,14 +84,15 @@ function GCSUtil(httpclient, filestream) { return requestConfig; } }); - - const endPoint = this.getGCSCustomEndPoint(stageInfo); - const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors }); + + const storage = Util.exists(endPoint) ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors }); client = { gcsToken: gcsToken, gcsClient: storage }; } else { client = null; } + process.nextTick(() => this.setupHttpClient(endPoint)); + return client; }; @@ -142,14 +155,14 @@ function GCSUtil(httpclient, filestream) { let matDescKey; try { - if (shouldPerformGCPBucket(accessToken)) { + if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) { const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']); - - const metadata = await meta['client'].gcsClient + const metadata = await meta['client'].gcsClient .bucket(gcsLocation.bucketName) .file(gcsLocation.path + filename) .getMetadata(); + digest = metadata[0].metadata[SFC_DIGEST]; contentLength = metadata[0].size; encryptionDataProp = metadata[0].metadata[ENCRYPTIONDATAPROP]; @@ -282,9 +295,9 @@ function GCSUtil(httpclient, filestream) { } try { - if (shouldPerformGCPBucket(accessToken)) { + if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) { const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']); - + await meta['client'].gcsClient .bucket(gcsLocation.bucketName) .file(gcsLocation.path + meta['dstFileName']) @@ -355,9 +368,8 @@ function GCSUtil(httpclient, filestream) { let size; try { - if (shouldPerformGCPBucket(accessToken)) { + if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) { const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']); - await meta['client'].gcsClient .bucket(gcsLocation.bucketName) .file(gcsLocation.path + meta['srcFileName']) @@ -376,9 +388,7 @@ function GCSUtil(httpclient, filestream) { size = metadata[0].size; } else { let response; - await axios({ - method: 'get', - url: downloadUrl, + await axios.get(downloadUrl, { headers: gcsHeaders, responseType: 'stream' }).then(async (res) => { @@ -466,6 +476,25 @@ function GCSUtil(httpclient, filestream) { } return endPoint; }; + + this.setupHttpClient = function (endPoint) { + if (typeof httpClient === 'undefined') { + const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'GCS Util'); + + //When http_proxy is enabled, the driver should use Axios for HTTPS requests to avoid relying on HTTP_PROXY in GCS. + if (proxy || Util.getEnvVar('http_proxy')) { + isProxyEnabled = true; + const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || 'storage.googleapis.com'); + axios = require('axios').create({ + proxy: false, + httpAgent: proxyAgent, + httpsAgent: proxyAgent, + }); + } else { + axios = require('axios'); + } + } + }; } module.exports = GCSUtil; diff --git a/lib/file_transfer_agent/remote_storage_util.js b/lib/file_transfer_agent/remote_storage_util.js index ffc599ec8..64741cb07 100644 --- a/lib/file_transfer_agent/remote_storage_util.js +++ b/lib/file_transfer_agent/remote_storage_util.js @@ -33,6 +33,8 @@ exports.SnowflakeFileEncryptionMaterial = SnowflakeFileEncryptionMaterial; * @constructor */ function RemoteStorageUtil(connectionConfig) { + let client = null; + /** * Get storage type based on location type. * @@ -41,15 +43,17 @@ function RemoteStorageUtil(connectionConfig) { * @returns {Object} */ this.getForStorageType = function (type) { + if (client) { + return client; + } if (type === 'S3') { - return new SnowflakeS3Util(connectionConfig); + client = new SnowflakeS3Util(connectionConfig); } else if (type === 'AZURE') { - return new SnowflakeAzureUtil(connectionConfig); + client = new SnowflakeAzureUtil(connectionConfig); } else if (type === 'GCS') { - return new SnowflakeGCSUtil(); - } else { - return null; + client = new SnowflakeGCSUtil(connectionConfig); } + return client; }; /** diff --git a/lib/proxy_util.js b/lib/proxy_util.js index e4097b6dc..5f2b87892 100644 --- a/lib/proxy_util.js +++ b/lib/proxy_util.js @@ -8,6 +8,17 @@ const Util = require('./util'); const GlobalConfig = require('./global_config'); const LoggingUtil = require('./logger/logging_util'); const ErrorCodes = Errors.codes; + +/** + * @typedef {object} Proxy + * @property {string} host - The host address of the proxy. + * @property {string} protocol - The protocol used by the proxy (e.g., "http" or "https") + * @property {string} user - The username for the proxy + * @property {number} port - The port number. + * @property {string} password - The password for the proxy + * @property {string} noProxy - Optional list of domains that should bypass the prox + */ + /** * remove http:// or https:// from the input, e.g. used with proxy URL * @param input @@ -25,8 +36,8 @@ exports.removeScheme = function (input) { * Return with the log constructed from the components detection and comparison * If there's something to warn the user about, return that too * - * @param the agentOptions object from agent creation - * @returns {object} + * @param {Proxy} agentOptions from agent creation + * @returns {{messages: string, warnings: string}} log messages */ exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { const envProxy = {}; @@ -55,20 +66,20 @@ exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { ' protocol=' + agentOptions.protocol + ' proxy=' + proxyHostAndPort : ' proxy=' + proxyHostAndPort; const proxyUsername = agentOptions.user ? ' user=' + agentOptions.user : ''; + const proxyString = `${Util.exists(agentOptions.user) ? `${agentOptions.user}:${agentOptions.password}@` : ''}${proxyHostAndPort}`.toLowerCase(); logMessages.messages = logMessages.messages + ` // Proxy configured in Agent:${proxyProtocolHostAndPort}${proxyUsername}`; // check if both the PROXY envvars and Connection proxy config is set // generate warnings if they are, and are also different if (envProxy.httpProxy && - this.removeScheme(envProxy.httpProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${envProxy.httpProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + this.removeScheme(envProxy.httpProxy).toLowerCase() !== proxyString.toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${this.describeProxy(this.getProxyFromEnv(false))})` + + ` and the Connection proxy (${this.describeProxy(agentOptions)}), but with different values.` + ' If you experience connectivity issues, try unsetting one of them.'; } - if (envProxy.httpsProxy && - this.removeScheme(envProxy.httpsProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${envProxy.httpsProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + if (envProxy.httpsProxy && this.removeScheme(envProxy.httpsProxy).toLowerCase() !== proxyString.toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${this.describeProxy(this.getProxyFromEnv(true))})` + + ` and the Connection proxy (${this.describeProxy(agentOptions)}) settings to connect, but with different values.` + ' If you experience connectivity issues, try unsetting one of them.'; } } @@ -77,6 +88,12 @@ exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { return logMessages; }; +/** + * Validate whether the proxy object has the appropriate information + * + * @param {Proxy} proxy + * @returns {Proxy} + */ exports.validateProxy = function (proxy) { const { host, port, noProxy, user, password } = proxy; // check for missing proxyHost @@ -123,11 +140,13 @@ exports.validateProxy = function (proxy) { delete proxy.password; } }; - -exports.validateEmptyString = function (value) { - return value !== '' ? value : undefined; -}; - + +/** + * Obtain the proxy information from the environment variable. + * + * @param {boolean} isHttps + * @returns {Proxy} + */ exports.getProxyFromEnv = function (isHttps = true) { const getDefaultPortIfNotSet = (proxyFromEnv) => { const isProxyProtocolHttps = proxyFromEnv.protocol === 'https:'; @@ -137,6 +156,7 @@ exports.getProxyFromEnv = function (isHttps = true) { return proxyFromEnv.port; } }; + const protocol = isHttps ? 'https' : 'http'; let proxyFromEnv = Util.getEnvVar(`${protocol}_proxy`); if (!proxyFromEnv){ @@ -161,7 +181,12 @@ exports.getProxyFromEnv = function (isHttps = true) { this.validateProxy(proxy); return proxy; }; - + +/** + * Obtain the no proxy information from the environment variable. + * + * @returns {string | undefined} + */ exports.getNoProxyEnv = function () { const noProxy = Util.getEnvVar('no_proxy'); if (noProxy) { @@ -170,6 +195,12 @@ exports.getNoProxyEnv = function () { return undefined; }; +/** + * Extract the host from the destination URL to check whether the same agent already exists or not. + * + * @param {string} destination + * @returns {string} + */ exports.getHostFromURL = function (destination) { if (destination.indexOf('://') === -1) { destination = 'https' + '://' + destination; @@ -183,16 +214,30 @@ exports.getHostFromURL = function (destination) { } }; -exports.getProxy = function (proxy, fileLocation, isHttps) { +/** + * if proxy exists, return the proxy. If not and the useEnvProxy is true, return the proxy from the environment variable. + * @param {Proxy} proxy + * @param {string} moduleName + * @param {string} isHttp + * + * @returns {Proxy} + */ +exports.getProxy = function (proxy, moduleName, isHttps) { if (!proxy && GlobalConfig.isEnvProxyActive()) { proxy = this.getProxyFromEnv(isHttps); if (proxy) { - Logger.getInstance().debug(`${fileLocation} loads the proxy info from the environment variable host: ${proxy.host}`); + Logger.getInstance().debug(`${moduleName} loads the proxy info from the environment variable host: ${proxy.host}`); } } return proxy; }; +/** + * The proxy configuration fields in Azure are different from the proxy fields in the snowflake node.js driver. + * Because of that, this function converts the snowflake proxy info to the Azure proxy info. + * @param {Proxy} proxy + * @returns {{host:string, port:number, user?:string, password?:string}}} + */ exports.getAzureProxy = function (proxy) { const AzureProxy = { ...proxy, host: `${proxy.protocol}${(proxy.protocol).endsWith(':') ? '' : ':'}//${proxy.host}`, @@ -266,8 +311,31 @@ exports.restoreEnvironmentProxy = function () { Logger.getInstance().debug('An Azure client has been created. Restore back the proxy environment variable values'); }; +/** + * Provide the details of the proxy info. + * @param proxy + * @returns {string} + */ exports.describeProxy = function (proxy) { - return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, ` - + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, ` - + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`; + if (Util.exists(proxy)) { + return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, ` + + `${Util.exists(proxy.user) ? `proxyUser: ${proxy.user}, proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, ` : ''}` + + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`; + } else { + return 'proxy was not configured'; + } +}; + +/** + * Make the proxy string with the proxy info (json format) + * @param proxy + * @returns {string} + */ +exports.stringifyProxy = function (proxy) { + if (Util.isEmptyObject(proxy)) { + return null; + } + return `${(proxy.protocol).startsWith('https') ? 'https' : 'http'}://` + + `${Util.exists(proxy.user) ? `${proxy.user}:${proxy.password}@` : ''}` + + `${proxy.host}:${proxy.port}`; }; \ No newline at end of file diff --git a/lib/util.js b/lib/util.js index b1d7830f1..526284f35 100644 --- a/lib/util.js +++ b/lib/util.js @@ -750,6 +750,20 @@ exports.isNotEmptyAsString = function (variable) { } return exports.exists(variable); }; +/** + * Checks Whether the object is empty (can be null or undefined) or not. + * @param object + * @returns {boolean} + */ +exports.isEmptyObject = (object) => { + if (!this.exists(object)) { + return true; + } + if (typeof object !== 'object') { + return false; + } + return Object.keys(object).length === 0; +}; exports.isWindows = function () { return os.platform() === 'win32'; diff --git a/test/unit/file_transfer_agent/gcs_test.js b/test/unit/file_transfer_agent/gcs_test.js index fd2ff96ad..1d7f7edbe 100644 --- a/test/unit/file_transfer_agent/gcs_test.js +++ b/test/unit/file_transfer_agent/gcs_test.js @@ -18,10 +18,17 @@ describe('GCS client', function () { const mockIv = 'mockIv'; const mockMatDesc = 'mockMatDesc'; const mockPresignedUrl = 'mockPresignedUrl'; + const connectionConfig = { + proxy: {}, + getProxy: function () { + return this.proxy; + }, + accessUrl: 'http://fakeaccount.snowflakecomputing.com', + }; let GCS; - let httpclient; - let filestream; + let httpClient; + let fileStream; const dataFile = mockDataFile; let meta; const encryptionMetadata = { @@ -44,7 +51,7 @@ describe('GCS client', function () { client: mockClient }; - mock('httpclient', { + mock('httpClient', { put: async function () { return; }, @@ -57,14 +64,14 @@ describe('GCS client', function () { }; } }); - mock('filestream', { + mock('fileStream', { readFileSync: async function (data) { return data; } }); - httpclient = require('httpclient'); - filestream = require('filestream'); - GCS = new SnowflakeGCSUtil(httpclient, filestream); + httpClient = require('httpClient'); + fileStream = require('fileStream'); + GCS = new SnowflakeGCSUtil(connectionConfig, httpClient, fileStream); }); describe('GCS client endpoint testing', async function () { @@ -144,7 +151,7 @@ describe('GCS client', function () { }); it('extract bucket name and path', async function () { - const GCS = new SnowflakeGCSUtil(); + const GCS = new SnowflakeGCSUtil(connectionConfig); let result = GCS.extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/'); assert.strictEqual(result.bucketName, 'sfc-eng-regression'); @@ -175,7 +182,7 @@ describe('GCS client', function () { }); it('get file header - fail not found file with presigned url', async function () { - mock('httpclient', { + mock('httpClient', { put: async function () { return; }, @@ -185,23 +192,23 @@ describe('GCS client', function () { throw err; } }); - const httpclient = require('httpclient'); - const GCS = new SnowflakeGCSUtil(httpclient); + const httpClient = require('httpClient'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient); await GCS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.NOT_FOUND_FILE); }); it('get file header - fail need retry', async function () { - mock('httpclient', { + mock('httpClient', { head: async function () { const err = new Error(); err.response = { status: 403 }; throw err; } }); - const httpclient = require('httpclient'); - const GCS = new SnowflakeGCSUtil(httpclient); + const httpClient = require('httpClient'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient); meta.presignedUrl = ''; @@ -210,15 +217,15 @@ describe('GCS client', function () { }); it('get file header - fail not found file without presigned url', async function () { - mock('httpclient', { + mock('httpClient', { head: async function () { const err = new Error(); err.response = { status: 404 }; throw err; } }); - const httpclient = require('httpclient'); - const GCS = new SnowflakeGCSUtil(httpclient); + const httpClient = require('httpClient'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient); meta.presignedUrl = ''; @@ -227,15 +234,15 @@ describe('GCS client', function () { }); it('get file header - fail expired token', async function () { - mock('httpclient', { + mock('httpClient', { head: async function () { const err = new Error(); err.response = { status: 401 }; throw err; } }); - const httpclient = require('httpclient'); - const GCS = new SnowflakeGCSUtil(httpclient); + const httpClient = require('httpClient'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient); meta.presignedUrl = ''; @@ -245,15 +252,15 @@ describe('GCS client', function () { it('get file header - fail unknown status', async function () { let err; - mock('httpclient', { + mock('httpClient', { head: async function () { err = new Error(); err.response = { status: 0 }; throw err; } }); - const httpclient = require('httpclient'); - const GCS = new SnowflakeGCSUtil(httpclient); + const httpClient = require('httpClient'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient); meta.presignedUrl = ''; @@ -270,42 +277,42 @@ describe('GCS client', function () { }); it('upload - fail need retry', async function () { - mock('httpclient', { + mock('httpClient', { put: async function () { const err = new Error(); err.code = 403; throw err; } }); - mock('filestream', { + mock('fileStream', { readFileSync: async function (data) { return data; } }); - httpclient = require('httpclient'); - filestream = require('filestream'); - const GCS = new SnowflakeGCSUtil(httpclient, filestream); + httpClient = require('httpClient'); + fileStream = require('fileStream'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient, fileStream); await GCS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.NEED_RETRY); }); it('upload - fail renew presigned url', async function () { - mock('httpclient', { + mock('httpClient', { put: async function () { const err = new Error(); err.code = 400; throw err; } }); - mock('filestream', { + mock('fileStream', { readFileSync: async function (data) { return data; } }); - httpclient = require('httpclient'); - filestream = require('filestream'); - const GCS = new SnowflakeGCSUtil(httpclient, filestream); + httpClient = require('httpClient'); + fileStream = require('fileStream'); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient, fileStream); meta.client = ''; meta.lastError = { code: 0 }; @@ -315,14 +322,14 @@ describe('GCS client', function () { }); it('upload - fail expired token', async function () { - mock('httpclient', { + mock('httpClient', { put: async function () { const err = new Error(); err.code = 401; throw err; } }); - mock('filestream', { + mock('fileStream', { readFileSync: async function (data) { return data; } @@ -344,10 +351,10 @@ describe('GCS client', function () { return new bucket; } }); - httpclient = require('httpclient'); - filestream = require('filestream'); + httpClient = require('httpClient'); + fileStream = require('fileStream'); const gcsClient = require('gcsClient'); - const GCS = new SnowflakeGCSUtil(httpclient, filestream); + const GCS = new SnowflakeGCSUtil(connectionConfig, httpClient, fileStream); meta.presignedUrl = ''; meta.client = { gcsToken: mockAccessToken, gcsClient: gcsClient }; @@ -355,4 +362,4 @@ describe('GCS client', function () { await GCS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); }); -}); +}); \ No newline at end of file diff --git a/test/unit/proxy_util_test.js b/test/unit/proxy_util_test.js index 55c9d57d0..f01b1c2b4 100644 --- a/test/unit/proxy_util_test.js +++ b/test/unit/proxy_util_test.js @@ -29,6 +29,19 @@ describe('ProxyUtil Test - removing http or https from string', () => { }); describe('ProxyUtil Test - detecting PROXY envvars and compare with the agent proxy settings', () => { + let originalHttpProxy = null; + let originalHttpsProxy = null; + + before(() => { + originalHttpProxy = process.env.HTTP_PROXY; + originalHttpsProxy = process.env.HTTPS_PROXY; + }); + + after(() => { + originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; + originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; + }); + [ { name: 'detect http_proxy envvar, no agent proxy', @@ -86,14 +99,22 @@ describe('ProxyUtil Test - detecting PROXY envvars and compare with the agent pr httpproxy: '', HTTPSPROXY: 'http://pro.xy:3128', agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + shouldLog: ' Using both the HTTPS_PROXY (proxyHost: pro.xy, proxyPort: 3128, proxyProtocol: http:, noProxy: undefined) and the Connection proxy (proxyHost: 10.20.30.40, proxyPort: 8080, proxyProtocol: undefined, noProxy: undefined) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' }, { name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an unauthenticated proxy, different from the envvars', isWarn: true, httpproxy: '169.254.169.254:8080', HTTPSPROXY: 'http://pro.xy:3128', agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTP_PROXY (169.254.169.254:8080) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + shouldLog: ' Using both the HTTP_PROXY (proxyHost: 169.254.169.254, proxyPort: 8080, proxyProtocol: http:, noProxy: undefined) and the Connection proxy (proxyHost: 10.20.30.40, proxyPort: 8080, proxyProtocol: undefined, noProxy: undefined), but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (proxyHost: pro.xy, proxyPort: 3128, proxyProtocol: http:, noProxy: undefined) and the Connection proxy (proxyHost: 10.20.30.40, proxyPort: 8080, proxyProtocol: undefined, noProxy: undefined) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + }, + { + name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an authenticated proxy, different from the envvars', + isWarn: true, + httpproxy: 'abc:def@169.254.169.254:8080', + HTTPSPROXY: 'http://cde:fge@pro.xy:3128', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'user': 'cde', 'password': 'fge', 'port': 8080 }, + shouldLog: ' Using both the HTTP_PROXY (proxyHost: 169.254.169.254, proxyPort: 8080, proxyUser: abc, proxyPassword is provided, proxyProtocol: http:, noProxy: undefined) and the Connection proxy (proxyHost: 10.20.30.40, proxyPort: 8080, proxyUser: cde, proxyPassword is provided, proxyProtocol: undefined, noProxy: undefined), but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (proxyHost: pro.xy, proxyPort: 3128, proxyUser: cde, proxyPassword is provided, proxyProtocol: http:, noProxy: undefined) and the Connection proxy (proxyHost: 10.20.30.40, proxyPort: 8080, proxyUser: cde, proxyPassword is provided, proxyProtocol: undefined, noProxy: undefined) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' } ].forEach(({ name, isWarn, httpproxy, HTTPSPROXY, agentOptions, shouldLog }) => { it(`${name}`, () => { diff --git a/test/unit/util_test.js b/test/unit/util_test.js index 107e7ac57..67d14cffc 100644 --- a/test/unit/util_test.js +++ b/test/unit/util_test.js @@ -751,7 +751,7 @@ describe('Util', function () { result: true, }, { - name: 'test - max retry timout is 0', + name: 'test - max retry timeout is 0', retryOption: { maxRetryCount: 7, numRetries: 1, @@ -781,7 +781,7 @@ describe('Util', function () { result: false, }, { - name: 'test - the remaining timout is 0', + name: 'test - the remaining timeout is 0', retryOption: { maxRetryCount: 7, numRetries: 8, @@ -791,7 +791,7 @@ describe('Util', function () { result: false, }, { - name: 'test - the remaining timoue is negative', + name: 'test - the remaining timeout is negative', retryOption: { maxRetryCount: 7, numRetries: 8, @@ -992,7 +992,7 @@ describe('Util', function () { result: false, }, { - name: 'credential manager is an empty obejct', + name: 'credential manager is an empty object', credentialManager: {}, result: false, }, @@ -1165,19 +1165,19 @@ describe('Util', function () { result: true, }, { - name: 'test - when the disableGCPTokenUplaod is enabled', + name: 'test - when the disableGCPTokenUpload is enabled', accessToken: 'Token', forceGCPUseDownscopedCredential: true, result: false, }, { - name: 'test - when token is empty but the disableGCPTokenUplaod is enabled', + name: 'test - when token is empty but the disableGCPTokenUpload is enabled', accessToken: null, forceGCPUseDownscopedCredential: true, result: false, }, { - name: 'test - test - when token is empty but the disableGCPTokenUplaod is disabled', + name: 'test - when token is empty but the disableGCPTokenUpload is disabled', accessToken: null, forceGCPUseDownscopedCredential: false, result: false, @@ -1215,33 +1215,88 @@ describe('Util', function () { } }); - describe('lstrip function Test', function () { + describe('isEmptyObject function test', function () { const testCases = [ { - name: 'remove consecutive characters /', - str: '///////////helloworld', - remove: '/', - result: 'helloworld' + name: 'JSON is not empty', + value: { 'hello': 'a' }, + result: false + }, + { + name: 'JSON is empty', + value: {}, + result: true + }, + { + name: 'non object(string)', + value: 'hello world', + result: false, + }, + { + name: 'non object(int)', + value: 123, + result: false, }, { - name: 'when the first character is not matched with the remove character', - str: '/\\/\\helloworld', - remove: '\\', - result: '/\\/\\helloworld' + name: 'non object(int)', + value: 123, + result: false, + }, + { + name: 'array', + value: [1, 2, 3], + result: false, }, { - name: 'when the first and the third characters are matched', - str: '@1@12345helloworld', - remove: '@', - result: '1@12345helloworld' + name: 'empty array', + value: [], + result: true, + }, + { + name: 'null', + value: null, + result: true, + }, { + name: 'undefined', + value: undefined, + result: true, }, ]; - for (const { name, str, remove, result } of testCases) { - it(name, function () { - assert.strictEqual(Util.lstrip(str, remove), result); + testCases.forEach(({ name, value, result }) => { + it(name, function () { + assert.strictEqual(Util.isEmptyObject(value), result); }); - } - }); + }); + + describe('lstrip function Test', function () { + const testCases = [ + { + name: 'remove consecutive characters /', + str: '///////////helloworld', + remove: '/', + result: 'helloworld' + }, + { + name: 'when the first character is not matched with the remove character', + str: '/\\/\\helloworld', + remove: '\\', + result: '/\\/\\helloworld' + }, + { + name: 'when the first and the third characters are matched', + str: '@1@12345helloworld', + remove: '@', + result: '1@12345helloworld' + }, + ]; -}); + for (const { name, str, remove, result } of testCases) { + it(name, function () { + assert.strictEqual(Util.lstrip(str, remove), result); + }); + } + }); + + }); +}); \ No newline at end of file