Merge pull request #12 from polarityio/develop

Add proxy support and improved msal-node logging

README.md

@@ -12,6 +12,13 @@ Check out the integration in action:
 
 Versions of this integration up to v3.4.1 supported authentication as a Sharepoint Add-in using a Client Secret. This method of authentication is discouraged by Microsoft and is no longer supported by this version of the integration. If you are using a previous version of the integration you will need to reconfigure the integration to use Azure App Authentication via Certificates. See the "Configuring Sharepoint" section below for more information.
 
+## Network Connectivity
+
+Note that to authenticate with Sharepoint in Azure, the integration will need access to both your Sharepoint site (e.g., https://mysite.sharepoint.com) and it will need access to the Microsoft authentication server
+`https://login.microsoftonline.com`.
+
+If your environment requires configuring a proxy, ensure that the proxy allows outbound connections to both your sharepoint site and `https://login.microsoftonline.com`.
+
 ## Configuring Sharepoint
 
 The Polarity-Sharepoint integration uses Azure App Authentication via Certificates. To configure the integration you register a new application with Azure. Once the application is registered you will need to upload a public certificate (the corresponding private certificate is needed by the integration on the Polarity Server). Finally, you need to set the appropriate API permissions. See below for detailed instructions. 
@@ -250,6 +257,26 @@ Error: EACCES: permission denied, open './certs/private.key'
 It means the public and or private key are not readable by the `polarityd` user. Ensure that the files are globally readable or are owned by the `polarityd` user.
 
 
+### Proxy Issues
+
+If the integration is unable to connect to your proxy or if the proxy is not allowing outbound connections to `login.microsoftonline.com` then you may see the following error:
+
+```
+ClientAuthError: endpoints_resolution_error: Error: could not resolve endpoints. Please check network and try again.
+```
+
+If you see this error, confirm that you have a proxy configured and that the configuration is correct. You can also test for connectivity from the Polarity Server command line using curl:
+
+```
+curl -vvv https://login.microsoftonline.com
+```
+
+If you are using a proxy then add the proxy flag (`-x`) and your proxy configuration string:
+
+```
+curl -vvv -x https://myproxy:8080 https://login.microsoftonline.com
+```
+
 ## Polarity
 
 Polarity is a memory-augmentation platform that improves and accelerates analyst decision making. For more information about the Polarity platform please see:

integration.js

@@ -8,6 +8,7 @@ const xbytes = require('xbytes');
 const NodeCache = require('node-cache');
 const msal = require('@azure/msal-node');
 const crypto = require('crypto');
+const SharepointHttpAuthClient = require('./sharepoint-http-client');
 
 const cache = new NodeCache({
  stdTTL: 60 * 10
@@ -191,6 +192,11 @@ function maybeSetClientApplication(options) {
  type: 'pkcs8'
  });
 
+ // The msal-node library uses its own HTTPClient which does not have proper proxy support.
+ // As a result, we have to implement our own HTTPClient that wraps the postman-request library
+ // to get proper proxy support.
+ const customHttpClient = new SharepointHttpAuthClient(requestWithDefaults, Logger);
+
  const clientConfig = {
  auth: {
  clientId: options.clientId,
@@ -199,13 +205,57 @@ function maybeSetClientApplication(options) {
  thumbprint: publicKeyThumbprint,
  privateKey
  }
+ },
+ system: {
+ loggerOptions: {
+ loggerCallback(logLevel, message, containsPii) {
+ Logger[msalLogLevelToPolarity(logLevel)]({ logLevel, message, containsPii }, 'MSAL Logger');
+ },
+ piiLoggingEnabled: config.logging.level === 'trace' ? true : false,
+ logLevel: polarityToMsalLogLevel(config.logging.level)
+ },
+ networkClient: customHttpClient.getClient()
  }
  };
 
+ Logger.trace({ clientConfig }, 'MSAL Client Config');
+
  clientApplication = new msal.ConfidentialClientApplication(clientConfig);
  }
 }
 
+function polarityToMsalLogLevel(polarityLogLevel) {
+ switch (polarityLogLevel) {
+ case 'trace':
+ return msal.LogLevel.Verbose;
+ case 'debug':
+ return msal.LogLevel.Verbose;
+ case 'info':
+ return msal.LogLevel.Info;
+ case 'warn':
+ return msal.LogLevel.Warning;
+ case 'error':
+ return msal.LogLevel.Error;
+ default:
+ return msal.LogLevel.Info;
+ }
+}
+
+function msalLogLevelToPolarity(msalLogLevel) {
+ switch (msalLogLevel) {
+ case msal.LogLevel.Verbose:
+ return 'trace';
+ case msal.LogLevel.Info:
+ return 'info';
+ case msal.LogLevel.Warning:
+ return 'warn';
+ case msal.LogLevel.Error:
+ return 'error';
+ default:
+ return 'info';
+ }
+}
+
 async function getToken(options) {
  const clientCredentialRequest = {
  //scopes: ['https://graph.microsoft.com/.default'] //clientCredentialRequestScopes,
@@ -232,6 +282,7 @@ async function getAuthToken(options) {
  }
 
  maybeSetClientApplication(options);
+
  let newToken = await getToken(options);
  cache.set(tokenCacheKey, newToken);
  return newToken;
@@ -260,12 +311,13 @@ function querySharepoint(entity, token, options, callback) {
 
  let totalRetriesLeft = 4;
  const requestSharepoint = () => {
- requestWithDefaults(requestOptions, (err, { statusCode, headers }, body) => {
+ requestWithDefaults(requestOptions, (err, response, body) => {
  if (err) return callback(err);
 
+ let { statusCode, headers } = response;
  const retryAfter = headers['Retry-After'] || headers['retry-after'];
  if (statusCode === 200) {
- Logger.trace({ headers }, 'Results of Sharepoint query headers');
+ //Logger.trace({ headers }, 'Results of Sharepoint query headers');
 
  callback(null, body);
  } else if ([429, 500, 503].includes(statusCode) && totalRetriesLeft) {
@@ -284,8 +336,6 @@ function querySharepoint(entity, token, options, callback) {
 const parseErrorToReadableJSON = (error) => JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error)));
 
 async function doLookup(entities, options, callback) {
- Logger.trace('starting lookup');
-
  options.subsite = options.subsite.startsWith('/') ? options.subsite.slice(1) : options.subsite;
 
  Logger.trace({ options }, 'doLookup options');
@@ -314,7 +364,7 @@ async function doLookup(entities, options, callback) {
  querySharepoint(entity, token, options, async (err, body) => {
  if (err) return done(err);
 
- Logger.trace({ entity, body }, 'Results of Sharepoint query');
+ //Logger.trace({ entity, body }, 'Results of Sharepoint query');
 
  if (body.PrimaryQueryResult.RelevantResults.RowCount < 1) return done(null, { entity, data: null });
 
@@ -401,7 +451,7 @@ function validateOptions(options, callback) {
  validateStringOption(errors, options, 'tenantId', 'You must provide a Tenant ID option.');
  validateStringOption(errors, options, 'publicKeyPath', 'You must provide a public key file path.');
  validateStringOption(errors, options, 'privateKeyPath', 'You must provide a private key file path.');
- 
+
  const subsiteStartWithError =
  options.subsite.value && options.subsite.value.startsWith('//')
  ? {

package.json

@@ -1,15 +1,15 @@
 {
  "name": "sharepoint",
- "version": "3.5.1",
+ "version": "3.5.2",
  "main": "./integration.js",
  "private": true,
  "license": "MIT",
  "author": "Polarity",
  "dependencies": {
- "async": "^3.2.4",
+ "async": "^3.2.5",
  "node-cache": "^5.1.2",
  "postman-request": "^2.88.1-postman.33",
  "xbytes": "^1.8.0",
- "@azure/msal-node": "1.18.3"
+ "@azure/msal-node": "1.18.4"
  }
 }

sharepoint-http-client.js

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2023, Polarity.io, Inc.
+ */
+
+/**
+ * Implements the msal-node `networkClient` interface using `postman-request` as the underlying HTTP client so that
+ * proxy support is available.
+ * See: https://github.com/AzureAD/microsoft-authentication-library-for-js/issues/2600#issuecomment-831299149
+ *
+ * The msal-node library originally used gaxios which does not have proper proxy support. To work around this
+ * msal-node implemented their own `networkClient` interface which should support proxies but it does not always
+ * work.
+ */
+class SharepointHttpAuthClient {
+ constructor(requestWithDefaults, Logger) {
+ this.requestWithDefaults = requestWithDefaults;
+ this.logger = Logger;
+ }
+
+ getClient() {
+ return {
+ sendGetRequestAsync: this.sendGetRequestAsync.bind(this),
+ sendPostRequestAsync: this.sendPostRequestAsync.bind(this)
+ };
+ }
+
+ async sendGetRequestAsync(uri, options) {
+ this.logger.trace({ uri, options }, 'sendGetRequestAsync');
+ return this.sendAsyncRequest(uri, 'GET', options);
+ }
+
+ async sendPostRequestAsync(uri, options) {
+ this.logger.trace({ uri, options }, 'sendPostRequestAsync');
+ return this.sendAsyncRequest(uri, 'POST', options);
+ }
+
+ async sendAsyncRequest(uri, method, options) {
+ return new Promise((resolve, reject) => {
+ const requestOptions = {
+ uri,
+ method,
+ json: true,
+ ...options
+ };
+ this.requestWithDefaults(requestOptions, (err, response, body) => {
+ if (err) {
+ return reject(err);
+ }
+
+ resolve({
+ headers: response.headers,
+ body,
+ status: response.status
+ });
+ });
+ });
+ }
+}
+
+module.exports = SharepointHttpAuthClient;