From 7e9a056d2cc1949729b8d079856d78531649fb48 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 9 Jan 2025 12:50:59 +0300 Subject: [PATCH 1/5] re-organize helpers and util --- packages/http/src/Adaptor.js | 2 +- packages/http/src/helpers.js | 146 ++++++++++++++++++++++++++++++++++ packages/http/src/util.js | 148 +---------------------------------- 3 files changed, 151 insertions(+), 145 deletions(-) create mode 100644 packages/http/src/helpers.js diff --git a/packages/http/src/Adaptor.js b/packages/http/src/Adaptor.js index 8d1afb9e4..21ff26e06 100644 --- a/packages/http/src/Adaptor.js +++ b/packages/http/src/Adaptor.js @@ -1,5 +1,5 @@ import { execute as commonExecute } from '@openfn/language-common'; -import { request as sendRequest, xmlParser } from './util'; +import { request as sendRequest, xmlParser } from './helpers'; /** * Options provided to the HTTP request diff --git a/packages/http/src/helpers.js b/packages/http/src/helpers.js new file mode 100644 index 000000000..ce9d8ff86 --- /dev/null +++ b/packages/http/src/helpers.js @@ -0,0 +1,146 @@ +import { composeNextState } from '@openfn/language-common'; +import { + request as commonRequest, + makeBasicAuthHeader, + expandReferences, + logResponse, +} from '@openfn/language-common/util'; +import * as cheerio from 'cheerio'; +import cheerioTableparser from 'cheerio-tableparser'; + +export function addAuth(configuration, headers) { + if (headers.Authorization) { + return; + } + + const { username, password, access_token } = configuration ?? {}; + + if (access_token) { + Object.assign(headers, { Authorization: `Bearer ${access_token}` }); + } else if (username && password) { + Object.assign(headers, makeBasicAuthHeader(username, password)); + } +} + +function encodeFormBody(data) { + const form = new FormData(); + for (const [key, value] of Object.entries(data)) { + form.append(key, value); + } + return form; +} + +const assertUrl = (pathOrUrl, baseUrl) => { + if (!baseUrl && pathOrUrl && !/^https?:\/\//.test(pathOrUrl)) { + const e = new Error('UNEXPECTED_RELATIVE_URL'); + e.code = 'UNEXPECTED_RELATIVE_URL'; + e.description = `You passed a relative URL but didn't set baseUrl`; + e.url = pathOrUrl; + e.fix = `Set the baseUrl on state.configuration or use an absolute URL, like https://example.com/api/${pathOrUrl}`; + throw e; + } + if (!baseUrl && !pathOrUrl) { + const e = new Error('NO_URL'); + e.code = 'NO_URL'; + e.description = `No URL provided`; + e.fix = `Make sure to pass a URL string into the request. You may need to set a baseURL on state.configuration.`; + throw e; + } +}; + +/** + * Request helper function + * @function + * @private + */ +export function request(method, path, params, callback = s => s) { + return state => { + const [resolvedPath, resolvedParams = {}] = expandReferences( + state, + path, + params + ); + + let { body, headers = {} } = resolvedParams; + + if (resolvedParams.json) { + console.warn( + 'WARNING: The `json` option has been deprecated. Use `body` instead' + ); + body = resolvedParams.json; + } + + if (resolvedParams.form) { + body = encodeFormBody(resolvedParams.form); + } + + const baseUrl = state.configuration?.baseUrl; + + assertUrl(resolvedPath, baseUrl); + + if (baseUrl) { + addAuth(state.configuration, headers); + } + + const maxRedirections = + resolvedParams.maxRedirections ?? + (resolvedParams.followAllRedirects === false ? 0 : 5); + + const tls = resolvedParams.tls ?? resolvedParams.agentOptions; + + if (resolvedParams.agentOptions) { + console.warn( + 'WARNING: The `agentOptions` option has been deprecated. Use `tls` instead' + ); + } + + const options = { + ...resolvedParams, + headers, + baseUrl, + body, + tls, + maxRedirections, + }; + + return commonRequest(method, resolvedPath, options) + .then(response => { + logResponse(response); + + return { + ...composeNextState(state, response.body), + response, + }; + }) + .then(callback) + .catch(err => { + logResponse(err); + + throw err; + }); + }; +} +/** + * XML parser helper function + * @function + * @private + */ +export function xmlParser(body, script, callback = s => s) { + return state => { + const [resolvedBody] = expandReferences(state, body); + const $ = cheerio.load(resolvedBody); + cheerioTableparser($); + + if (script) { + const result = script($); + try { + const r = JSON.parse(result); + return callback(composeNextState(state, r)); + } catch (e) { + return callback(composeNextState(state, { body: result })); + } + } else { + return callback(composeNextState(state, { body: resolvedBody })); + } + }; +} diff --git a/packages/http/src/util.js b/packages/http/src/util.js index 5670bb0b2..2cea62962 100644 --- a/packages/http/src/util.js +++ b/packages/http/src/util.js @@ -1,144 +1,4 @@ -import { composeNextState } from '@openfn/language-common'; -import { - request as commonRequest, - makeBasicAuthHeader, - expandReferences, - logResponse, - encode, - decode, - uuid, -} from '@openfn/language-common/util'; - -import * as cheerio from 'cheerio'; -import cheerioTableparser from 'cheerio-tableparser'; - -export function addAuth(configuration, headers) { - if (headers.Authorization) { - return; - } - - const { username, password, access_token } = configuration ?? {}; - - if (access_token) { - Object.assign(headers, { Authorization: `Bearer ${access_token}` }); - } else if (username && password) { - Object.assign(headers, makeBasicAuthHeader(username, password)); - } -} - -function encodeFormBody(data) { - const form = new FormData(); - for (const [key, value] of Object.entries(data)) { - form.append(key, value); - } - return form; -} - -const assertUrl = (pathOrUrl, baseUrl) => { - if (!baseUrl && pathOrUrl && !/^https?:\/\//.test(pathOrUrl)) { - const e = new Error('UNEXPECTED_RELATIVE_URL'); - e.code = 'UNEXPECTED_RELATIVE_URL'; - e.description = `You passed a relative URL but didn't set baseUrl`; - e.url = pathOrUrl; - e.fix = `Set the baseUrl on state.configuration or use an absolute URL, like https://example.com/api/${pathOrUrl}`; - throw e; - } - if (!baseUrl && !pathOrUrl) { - const e = new Error('NO_URL'); - e.code = 'NO_URL'; - e.description = `No URL provided`; - e.fix = `Make sure to pass a URL string into the request. You may need to set a baseURL on state.configuration.`; - throw e; - } -}; - -export function request(method, path, params, callback = s => s) { - return state => { - const [resolvedPath, resolvedParams = {}] = expandReferences( - state, - path, - params - ); - - let { body, headers = {} } = resolvedParams; - - if (resolvedParams.json) { - console.warn( - 'WARNING: The `json` option has been deprecated. Use `body` instead' - ); - body = resolvedParams.json; - } - - if (resolvedParams.form) { - body = encodeFormBody(resolvedParams.form); - } - - const baseUrl = state.configuration?.baseUrl; - - assertUrl(resolvedPath, baseUrl); - - if (baseUrl) { - addAuth(state.configuration, headers); - } - - const maxRedirections = - resolvedParams.maxRedirections ?? - (resolvedParams.followAllRedirects === false ? 0 : 5); - - const tls = resolvedParams.tls ?? resolvedParams.agentOptions; - - if (resolvedParams.agentOptions) { - console.warn( - 'WARNING: The `agentOptions` option has been deprecated. Use `tls` instead' - ); - } - - const options = { - ...resolvedParams, - headers, - baseUrl, - body, - tls, - maxRedirections, - }; - - return commonRequest(method, resolvedPath, options) - .then(response => { - logResponse(response); - - return { - ...composeNextState(state, response.body), - response, - }; - }) - .then(callback) - .catch(err => { - logResponse(err); - - throw err; - }); - }; -} - -export function xmlParser(body, script, callback = s => s) { - return state => { - const [resolvedBody] = expandReferences(state, body); - const $ = cheerio.load(resolvedBody); - cheerioTableparser($); - - if (script) { - const result = script($); - try { - const r = JSON.parse(result); - return callback(composeNextState(state, r)); - } catch (e) { - return callback(composeNextState(state, { body: result })); - } - } else { - return callback(composeNextState(state, { body: resolvedBody })); - } - }; -} +import { encode, decode, uuid } from '@openfn/language-common/util'; export { /** @@ -148,7 +8,7 @@ export { * @param {string} data - The string to be encoded. * @returns {string} - The Base64 encoded string. * @example Encode a string - * const encoded = Util.encode('Hello World'); + * const encoded = util.encode('Hello World'); * console.log(encoded); // Output: SGVsbG8gV29ybGQ= */ encode, @@ -159,7 +19,7 @@ export { * @param {string} base64Data - The Base64 encoded string. * @returns {string} - The decoded string. * @example Decode a Base64 string - * const decoded = Util.decode('SGVsbG8gV29ybGQ='); + * const decoded = util.decode('SGVsbG8gV29ybGQ='); * console.log(decoded); // Output: Hello World */ decode, @@ -169,7 +29,7 @@ export { * @public * @returns {string} - A newly generated UUID. * @example Generate a UUID - * const id = Util.uuid(); + * const id = util.uuid(); * console.log(id); // Output:'3f4e254e-8f6f-4f8b-9651-1c1c262cc83f' */ uuid, From 33c11f14f2d001cb2b55d50d6d27236e78f7f37a Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 9 Jan 2025 13:09:06 +0300 Subject: [PATCH 2/5] remove export from addAuth --- packages/http/src/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http/src/helpers.js b/packages/http/src/helpers.js index ce9d8ff86..7d15a42bc 100644 --- a/packages/http/src/helpers.js +++ b/packages/http/src/helpers.js @@ -8,7 +8,7 @@ import { import * as cheerio from 'cheerio'; import cheerioTableparser from 'cheerio-tableparser'; -export function addAuth(configuration, headers) { +function addAuth(configuration, headers) { if (headers.Authorization) { return; } From 4b9a5b9dcb6f469361baf14352dc6e9535319e1f Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 9 Jan 2025 13:10:32 +0300 Subject: [PATCH 3/5] add changeset --- .changeset/dull-peas-peel.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/dull-peas-peel.md diff --git a/.changeset/dull-peas-peel.md b/.changeset/dull-peas-peel.md new file mode 100644 index 000000000..9bbe8abc0 --- /dev/null +++ b/.changeset/dull-peas-peel.md @@ -0,0 +1,7 @@ +--- +'@openfn/language-http': patch +--- + +- Fix typo in util functions examples +- Move helper functions to `helper.js` +- Remove export for `addAuth()` helper From 99344c340fcc1c86526fec9755c511fe9c246773 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 9 Jan 2025 20:38:19 +0300 Subject: [PATCH 4/5] remove helpers.js --- packages/http/src/Adaptor.js | 2 +- packages/http/src/helpers.js | 146 ---------------------------------- packages/http/src/util.js | 150 ++++++++++++++++++++++++++++++++++- 3 files changed, 150 insertions(+), 148 deletions(-) delete mode 100644 packages/http/src/helpers.js diff --git a/packages/http/src/Adaptor.js b/packages/http/src/Adaptor.js index 21ff26e06..8d1afb9e4 100644 --- a/packages/http/src/Adaptor.js +++ b/packages/http/src/Adaptor.js @@ -1,5 +1,5 @@ import { execute as commonExecute } from '@openfn/language-common'; -import { request as sendRequest, xmlParser } from './helpers'; +import { request as sendRequest, xmlParser } from './util'; /** * Options provided to the HTTP request diff --git a/packages/http/src/helpers.js b/packages/http/src/helpers.js deleted file mode 100644 index 7d15a42bc..000000000 --- a/packages/http/src/helpers.js +++ /dev/null @@ -1,146 +0,0 @@ -import { composeNextState } from '@openfn/language-common'; -import { - request as commonRequest, - makeBasicAuthHeader, - expandReferences, - logResponse, -} from '@openfn/language-common/util'; -import * as cheerio from 'cheerio'; -import cheerioTableparser from 'cheerio-tableparser'; - -function addAuth(configuration, headers) { - if (headers.Authorization) { - return; - } - - const { username, password, access_token } = configuration ?? {}; - - if (access_token) { - Object.assign(headers, { Authorization: `Bearer ${access_token}` }); - } else if (username && password) { - Object.assign(headers, makeBasicAuthHeader(username, password)); - } -} - -function encodeFormBody(data) { - const form = new FormData(); - for (const [key, value] of Object.entries(data)) { - form.append(key, value); - } - return form; -} - -const assertUrl = (pathOrUrl, baseUrl) => { - if (!baseUrl && pathOrUrl && !/^https?:\/\//.test(pathOrUrl)) { - const e = new Error('UNEXPECTED_RELATIVE_URL'); - e.code = 'UNEXPECTED_RELATIVE_URL'; - e.description = `You passed a relative URL but didn't set baseUrl`; - e.url = pathOrUrl; - e.fix = `Set the baseUrl on state.configuration or use an absolute URL, like https://example.com/api/${pathOrUrl}`; - throw e; - } - if (!baseUrl && !pathOrUrl) { - const e = new Error('NO_URL'); - e.code = 'NO_URL'; - e.description = `No URL provided`; - e.fix = `Make sure to pass a URL string into the request. You may need to set a baseURL on state.configuration.`; - throw e; - } -}; - -/** - * Request helper function - * @function - * @private - */ -export function request(method, path, params, callback = s => s) { - return state => { - const [resolvedPath, resolvedParams = {}] = expandReferences( - state, - path, - params - ); - - let { body, headers = {} } = resolvedParams; - - if (resolvedParams.json) { - console.warn( - 'WARNING: The `json` option has been deprecated. Use `body` instead' - ); - body = resolvedParams.json; - } - - if (resolvedParams.form) { - body = encodeFormBody(resolvedParams.form); - } - - const baseUrl = state.configuration?.baseUrl; - - assertUrl(resolvedPath, baseUrl); - - if (baseUrl) { - addAuth(state.configuration, headers); - } - - const maxRedirections = - resolvedParams.maxRedirections ?? - (resolvedParams.followAllRedirects === false ? 0 : 5); - - const tls = resolvedParams.tls ?? resolvedParams.agentOptions; - - if (resolvedParams.agentOptions) { - console.warn( - 'WARNING: The `agentOptions` option has been deprecated. Use `tls` instead' - ); - } - - const options = { - ...resolvedParams, - headers, - baseUrl, - body, - tls, - maxRedirections, - }; - - return commonRequest(method, resolvedPath, options) - .then(response => { - logResponse(response); - - return { - ...composeNextState(state, response.body), - response, - }; - }) - .then(callback) - .catch(err => { - logResponse(err); - - throw err; - }); - }; -} -/** - * XML parser helper function - * @function - * @private - */ -export function xmlParser(body, script, callback = s => s) { - return state => { - const [resolvedBody] = expandReferences(state, body); - const $ = cheerio.load(resolvedBody); - cheerioTableparser($); - - if (script) { - const result = script($); - try { - const r = JSON.parse(result); - return callback(composeNextState(state, r)); - } catch (e) { - return callback(composeNextState(state, { body: result })); - } - } else { - return callback(composeNextState(state, { body: resolvedBody })); - } - }; -} diff --git a/packages/http/src/util.js b/packages/http/src/util.js index 2cea62962..7bfefdc27 100644 --- a/packages/http/src/util.js +++ b/packages/http/src/util.js @@ -1,4 +1,152 @@ -import { encode, decode, uuid } from '@openfn/language-common/util'; +import { composeNextState } from '@openfn/language-common'; +import { + request as commonRequest, + makeBasicAuthHeader, + expandReferences, + logResponse, + encode, + decode, + uuid, +} from '@openfn/language-common/util'; +import * as cheerio from 'cheerio'; +import cheerioTableparser from 'cheerio-tableparser'; + +function addAuth(configuration, headers) { + if (headers.Authorization) { + return; + } + + const { username, password, access_token } = configuration ?? {}; + + if (access_token) { + Object.assign(headers, { Authorization: `Bearer ${access_token}` }); + } else if (username && password) { + Object.assign(headers, makeBasicAuthHeader(username, password)); + } +} + +function encodeFormBody(data) { + const form = new FormData(); + for (const [key, value] of Object.entries(data)) { + form.append(key, value); + } + return form; +} + +const assertUrl = (pathOrUrl, baseUrl) => { + if (!baseUrl && pathOrUrl && !/^https?:\/\//.test(pathOrUrl)) { + const e = new Error('UNEXPECTED_RELATIVE_URL'); + e.code = 'UNEXPECTED_RELATIVE_URL'; + e.description = `You passed a relative URL but didn't set baseUrl`; + e.url = pathOrUrl; + e.fix = `Set the baseUrl on state.configuration or use an absolute URL, like https://example.com/api/${pathOrUrl}`; + throw e; + } + if (!baseUrl && !pathOrUrl) { + const e = new Error('NO_URL'); + e.code = 'NO_URL'; + e.description = `No URL provided`; + e.fix = `Make sure to pass a URL string into the request. You may need to set a baseURL on state.configuration.`; + throw e; + } +}; + +/** + * Request helper function + * @function + * @private + */ +export function request(method, path, params, callback = s => s) { + return state => { + const [resolvedPath, resolvedParams = {}] = expandReferences( + state, + path, + params + ); + + let { body, headers = {} } = resolvedParams; + + if (resolvedParams.json) { + console.warn( + 'WARNING: The `json` option has been deprecated. Use `body` instead' + ); + body = resolvedParams.json; + } + + if (resolvedParams.form) { + body = encodeFormBody(resolvedParams.form); + } + + const baseUrl = state.configuration?.baseUrl; + + assertUrl(resolvedPath, baseUrl); + + if (baseUrl) { + addAuth(state.configuration, headers); + } + + const maxRedirections = + resolvedParams.maxRedirections ?? + (resolvedParams.followAllRedirects === false ? 0 : 5); + + const tls = resolvedParams.tls ?? resolvedParams.agentOptions; + + if (resolvedParams.agentOptions) { + console.warn( + 'WARNING: The `agentOptions` option has been deprecated. Use `tls` instead' + ); + } + + const options = { + ...resolvedParams, + headers, + baseUrl, + body, + tls, + maxRedirections, + }; + + return commonRequest(method, resolvedPath, options) + .then(response => { + logResponse(response); + + return { + ...composeNextState(state, response.body), + response, + }; + }) + .then(callback) + .catch(err => { + logResponse(err); + + throw err; + }); + }; +} +/** + * XML parser helper function + * @function + * @private + */ +export function xmlParser(body, script, callback = s => s) { + return state => { + const [resolvedBody] = expandReferences(state, body); + const $ = cheerio.load(resolvedBody); + cheerioTableparser($); + + if (script) { + const result = script($); + try { + const r = JSON.parse(result); + return callback(composeNextState(state, r)); + } catch (e) { + return callback(composeNextState(state, { body: result })); + } + } else { + return callback(composeNextState(state, { body: resolvedBody })); + } + }; +} export { /** From 17d153448a21c8c8dc178d112ac0c9397b44cf64 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 9 Jan 2025 20:39:50 +0300 Subject: [PATCH 5/5] update changeset --- .changeset/dull-peas-peel.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.changeset/dull-peas-peel.md b/.changeset/dull-peas-peel.md index 9bbe8abc0..ce7f2da55 100644 --- a/.changeset/dull-peas-peel.md +++ b/.changeset/dull-peas-peel.md @@ -3,5 +3,4 @@ --- - Fix typo in util functions examples -- Move helper functions to `helper.js` - Remove export for `addAuth()` helper