From 507d3f583ddfd3a063e890d491e8060fe438b7b5 Mon Sep 17 00:00:00 2001 From: Alexander Zaslonov Date: Wed, 18 Nov 2020 14:12:04 -0800 Subject: [PATCH] Added better error handling and messaging for scripts.v2 (#1042) --- scripts.v2/capture.js | 84 ++++++++++++++++++++------------- scripts.v2/cleanup.js | 50 ++++++++++++++------ scripts.v2/generate.js | 38 +++++++++------ scripts.v2/migrate.bat | 8 ++++ scripts.v2/migrate.js | 54 ++++++++++++--------- scripts.v2/utils.js | 103 +++++++++++++++++++++++++---------------- scripts.v3/cleanup.js | 2 +- 7 files changed, 214 insertions(+), 125 deletions(-) create mode 100644 scripts.v2/migrate.bat diff --git a/scripts.v2/capture.js b/scripts.v2/capture.js index 47cfc516e..789f591bd 100644 --- a/scripts.v2/capture.js +++ b/scripts.v2/capture.js @@ -7,56 +7,76 @@ const destinationFolder = process.argv[4]; async function getContentTypes() { - const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes?api-version=2019-12-01`, managementApiAccessToken); - const contentTypes = data.value.map(x => x.id.replace("\/contentTypes\/", "")); + try { + const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes?api-version=2019-12-01`, managementApiAccessToken); + const contentTypes = data.value.map(x => x.id.replace("\/contentTypes\/", "")); - return contentTypes; + return contentTypes; + } + catch (error) { + throw new Error(`Unable to fetch content types. ${error.message}`); + } } async function getContentItems(contentType) { - const contentItems = []; - let nextPageUrl = `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes/${contentType}/contentItems?api-version=2019-12-01`; + try { + const contentItems = []; + let nextPageUrl = `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes/${contentType}/contentItems?api-version=2019-12-01`; - do { - const data = await request("GET", nextPageUrl, managementApiAccessToken); - contentItems.push(...data.value); + do { + const data = await request("GET", nextPageUrl, managementApiAccessToken); + contentItems.push(...data.value); - if (data.value.length > 0 && data.nextLink) { - nextPageUrl = data.nextLink; - } - else { - nextPageUrl = null; + if (data.value.length > 0 && data.nextLink) { + nextPageUrl = data.nextLink; + } + else { + nextPageUrl = null; + } } - } - while (nextPageUrl) + while (nextPageUrl) - return contentItems; + return contentItems; + } + catch (error) { + throw new Error(`Unable to fetch content items. ${error.message}`); + } } async function captureJson() { - const result = {}; - const contentTypes = await getContentTypes(); + try { + const result = {}; + const contentTypes = await getContentTypes(); - for (const contentType of contentTypes) { - const contentItems = await getContentItems(contentType); + for (const contentType of contentTypes) { + const contentItems = await getContentItems(contentType); - contentItems.forEach(contentItem => { - result[contentItem.id] = contentItem; - delete contentItem.id; - }); - } + contentItems.forEach(contentItem => { + result[contentItem.id] = contentItem; + delete contentItem.id; + }); + } - await fs.promises.mkdir(path.resolve(destinationFolder), { recursive: true }); + await fs.promises.mkdir(path.resolve(destinationFolder), { recursive: true }); - fs.writeFileSync(`${destinationFolder}/data.json`, JSON.stringify(result)); + fs.writeFileSync(`${destinationFolder}/data.json`, JSON.stringify(result)); + } + catch (error) { + throw new Error(`Unable to capture content. ${error.message}`); + } } async function capture() { - const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); - const localMediaFolder = `./${destinationFolder}/media`; + try { + const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); + const localMediaFolder = `./${destinationFolder}/media`; - await captureJson(); - await downloadBlobs(blobStorageUrl, localMediaFolder); + await captureJson(); + await downloadBlobs(blobStorageUrl, localMediaFolder); + } + catch (error) { + throw new Error(`Unable to complete export. ${error.message}`); + } } capture() @@ -64,5 +84,5 @@ capture() console.log("DONE"); }) .catch(error => { - console.log(error); + console.log(error.message); }); \ No newline at end of file diff --git a/scripts.v2/cleanup.js b/scripts.v2/cleanup.js index c72da1b71..6130c526b 100644 --- a/scripts.v2/cleanup.js +++ b/scripts.v2/cleanup.js @@ -4,36 +4,56 @@ const managementApiAccessToken = process.argv[3]; async function getContentTypes() { - const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes?api-version=2019-12-01`, managementApiAccessToken); - const contentTypes = data.value.map(x => x.id.replace("\/contentTypes\/", "")); + try { + const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes?api-version=2019-12-01`, managementApiAccessToken); + const contentTypes = data.value.map(x => x.id.replace("\/contentTypes\/", "")); - return contentTypes; + return contentTypes; + } + catch (error) { + throw new Error(`Unable to fetch content types. ${error.message}`); + } } async function getContentItems(contentType) { - const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes/${contentType}/contentItems?api-version=2019-12-01`, managementApiAccessToken); - const contentItems = data.value; + try { + const data = await request("GET", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/contentTypes/${contentType}/contentItems?api-version=2019-12-01`, managementApiAccessToken); + const contentItems = data.value; - return contentItems; + return contentItems; + } + catch (error) { + throw new Error(`Unable to fetch content items. ${error.message}`); + } } async function deleteContent() { - const contentTypes = await getContentTypes(); + try { + const contentTypes = await getContentTypes(); - for (const contentType of contentTypes) { - const contentItems = await getContentItems(contentType); + for (const contentType of contentTypes) { + const contentItems = await getContentItems(contentType); - for (const contentItem of contentItems) { - await request("DELETE", `https://${managementApiEndpoint}//subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/${contentItem.id}?api-version=2019-12-01`, managementApiAccessToken); + for (const contentItem of contentItems) { + await request("DELETE", `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/${contentItem.id}?api-version=2019-12-01`, managementApiAccessToken); + } } } + catch (error) { + throw new Error(`Unable to delete content. ${error.message}`); + } } async function cleanup() { - const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); + try { + const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); - await deleteContent(); - await deleteBlobs(blobStorageUrl); + await deleteContent(); + await deleteBlobs(blobStorageUrl); + } + catch (error) { + throw new Error(`Unable to complete cleanup. ${error.message}`); + } } cleanup() @@ -41,5 +61,5 @@ cleanup() console.log("DONE"); }) .catch(error => { - console.log(error); + console.log(error.message); }); \ No newline at end of file diff --git a/scripts.v2/generate.js b/scripts.v2/generate.js index 5e61299fe..7d0f8e48a 100644 --- a/scripts.v2/generate.js +++ b/scripts.v2/generate.js @@ -6,25 +6,35 @@ const sourceFolder = process.argv[4]; async function generateJson() { - const data = fs.readFileSync(`${sourceFolder}/data.json`); - const dataObj = JSON.parse(data); - const keys = Object.keys(dataObj); + try { + const data = fs.readFileSync(`${sourceFolder}/data.json`); + const dataObj = JSON.parse(data); + const keys = Object.keys(dataObj); - for (const key of keys) { - await request( - "PUT", - `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/${key}?api-version=2019-12-01`, - managementApiAccessToken, - JSON.stringify(dataObj[key])); + for (const key of keys) { + await request( + "PUT", + `https://${managementApiEndpoint}/subscriptions/00000/resourceGroups/00000/providers/Microsoft.ApiManagement/service/00000/${key}?api-version=2019-12-01`, + managementApiAccessToken, + JSON.stringify(dataObj[key])); + } + } + catch (error) { + throw new Error(`Unable to generate the content. ${error.message}`); } } async function generate() { - const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); - const localMediaFolder = `./${sourceFolder}/media`; + try { + const blobStorageUrl = await getStorageSasTokenOrThrow(managementApiEndpoint, managementApiAccessToken); + const localMediaFolder = `./${sourceFolder}/media`; - await generateJson(); - await uploadBlobs(blobStorageUrl, localMediaFolder); + await generateJson(); + await uploadBlobs(blobStorageUrl, localMediaFolder); + } + catch (error) { + throw new Error(`Unable to complete import. ${error.message}`); + } } generate() @@ -32,5 +42,5 @@ generate() console.log("DONE"); }) .catch(error => { - console.log(error); + console.log(error.message); }); \ No newline at end of file diff --git a/scripts.v2/migrate.bat b/scripts.v2/migrate.bat new file mode 100644 index 000000000..9ee803d56 --- /dev/null +++ b/scripts.v2/migrate.bat @@ -0,0 +1,8 @@ +@REM This script automates content migration between developer portal instances. + +node ./migrate ^ +--sourceEndpoint "" ^ +--sourceToken "" ^ +--destEndpoint "" ^ +--destToken "" ^ +--publishEndpoint "" diff --git a/scripts.v2/migrate.js b/scripts.v2/migrate.js index e1668c138..b6cb54f2a 100644 --- a/scripts.v2/migrate.js +++ b/scripts.v2/migrate.js @@ -100,31 +100,36 @@ const yargs = require('yargs') .argv; async function run() { - const sourceManagementApiEndpoint = yargs.sourceEndpoint; - const sourceManagementApiAccessToken = await getTokenOrThrow(yargs.sourceToken, yargs.sourceId, yargs.sourceKey); + try { + const sourceManagementApiEndpoint = yargs.sourceEndpoint; + const sourceManagementApiAccessToken = await getTokenOrThrow(yargs.sourceToken, yargs.sourceId, yargs.sourceKey); - const destManagementApiEndpoint = yargs.destEndpoint; - const destManagementApiAccessToken = await getTokenOrThrow(yargs.destToken, yargs.destId, yargs.destKey); - const publishEndpoint = yargs.publishEndpoint; + const destManagementApiEndpoint = yargs.destEndpoint; + const destManagementApiAccessToken = await getTokenOrThrow(yargs.destToken, yargs.destId, yargs.destKey); + const publishEndpoint = yargs.publishEndpoint; - // the rest of this mirrors migrate.bat, but since we're JS, we're platform-agnostic. - const snapshotFolder = '../dist/snapshot'; + // the rest of this mirrors migrate.bat, but since we're JS, we're platform-agnostic. + const snapshotFolder = '../dist/snapshot'; - // capture the content of the source portal - execSync(`node ./capture ${sourceManagementApiEndpoint} "${sourceManagementApiAccessToken}" "${snapshotFolder}"`); + // capture the content of the source portal + execSync(`node ./capture ${sourceManagementApiEndpoint} "${sourceManagementApiAccessToken}" "${snapshotFolder}"`); - // remove all content of the target portal - execSync(`node ./cleanup ${destManagementApiEndpoint} "${destManagementApiAccessToken}"`); + // remove all content of the target portal + execSync(`node ./cleanup ${destManagementApiEndpoint} "${destManagementApiAccessToken}"`); - // upload the content of the source portal - execSync(`node ./generate ${destManagementApiEndpoint} "${destManagementApiAccessToken}" "${snapshotFolder}"`); + // upload the content of the source portal + execSync(`node ./generate ${destManagementApiEndpoint} "${destManagementApiAccessToken}" "${snapshotFolder}"`); - if (publishEndpoint && !yargs.selfHosted) { - process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; - await publish(publishEndpoint, destManagementApiAccessToken); + if (publishEndpoint && !yargs.selfHosted) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; + await publish(publishEndpoint, destManagementApiAccessToken); + } + else if (publishEndpoint) { + console.warn("Auto-publishing self-hosted portal is not supported."); + } } - else if (publishEndpoint) { - console.warn("Auto-publishing self-hosted portal is not supported."); + catch (error) { + throw new Error(`Unable to complete migration. ${error.message}`); } } @@ -172,10 +177,15 @@ async function generateSASToken(id, key, expiresIn = 3600) { * @param {string} token the SAS token */ async function publish(endpoint, token) { - const url = `https://${endpoint}/publish`; + try { + const url = `https://${endpoint}/publish`; - // returns with literal OK (missing quotes), which is invalid json. - await request("POST", url, token); + // returns with literal OK (missing quotes), which is invalid json. + await request("POST", url, token); + } + catch (error) { + throw new Error(`Unable to schedule website publishing. ${error.message}`); + } } run() @@ -183,6 +193,6 @@ run() console.log("DONE"); }) .catch(error => { - console.error(error); + console.error(error.message); process.exitCode = 1; }); diff --git a/scripts.v2/utils.js b/scripts.v2/utils.js index 7517605e8..fd1c7a7a6 100644 --- a/scripts.v2/utils.js +++ b/scripts.v2/utils.js @@ -15,7 +15,7 @@ function listFilesInDirectory(dir) { const stat = fs.statSync(file); if (stat && stat.isDirectory()) { - results.push(...this.listAllFilesInDirectory(file)); + results.push(...listFilesInDirectory(file)); } else { results.push(file); } @@ -81,16 +81,22 @@ async function request(method, url, accessToken, body) { }); resp.on('end', () => { - try { - if (data && data.startsWith("{")) { - resolve(JSON.parse(data)); - } - else { - resolve(data); - } - } - catch (e) { - reject(e); + switch (resp.statusCode) { + case 200: + case 201: + data.startsWith("{") ? resolve(JSON.parse(data)) : resolve(data); + break; + case 404: + reject({ code: "NotFound", message: `Resource not found: ${requestUrl}` }); + break; + case 401: + reject({ code: "Unauthorized", message: `Unauthorized. Make sure you correctly specified management API access token before running the script.` }); + break; + case 403: + reject({ code: "Forbidden", message: `Looks like you are not allowed to perform this operation. Please check with your administrator.` }); + break; + default: + reject({ code: "UnhandledError", message: `Could not complete request to ${requestUrl}. Status: ${resp.statusCode} ${resp.statusMessage}` }); } }); }); @@ -108,54 +114,69 @@ async function request(method, url, accessToken, body) { } async function downloadBlobs(blobStorageUrl, localMediaFolder) { - const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); - const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); + try { + const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); + const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); - await fs.promises.mkdir(path.resolve(localMediaFolder), { recursive: true }); + await fs.promises.mkdir(path.resolve(localMediaFolder), { recursive: true }); - let blobs = containerClient.listBlobsFlat(); + let blobs = containerClient.listBlobsFlat(); - for await (const blob of blobs) { - const blockBlobClient = containerClient.getBlockBlobClient(blob.name); - const extension = mime.extension(blob.properties.contentType); + for await (const blob of blobs) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + const extension = mime.extension(blob.properties.contentType); - if (extension != null) { - await blockBlobClient.downloadToFile(`${localMediaFolder}/${blob.name}.${extension}`); - } - else { - await blockBlobClient.downloadToFile(`${localMediaFolder}/${blob.name}`); + if (extension != null) { + await blockBlobClient.downloadToFile(`${localMediaFolder}/${blob.name}.${extension}`); + } + else { + await blockBlobClient.downloadToFile(`${localMediaFolder}/${blob.name}`); + } } } + catch (error) { + throw new Error(`Unable to download media files. ${error.message}`); + } } async function uploadBlobs(blobStorageUrl, localMediaFolder) { - const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); - const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); - const fileNames = listFilesInDirectory(localMediaFolder); + try { + const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); + const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); + const fileNames = listFilesInDirectory(localMediaFolder); - for (const fileName of fileNames) { - const blobName = path.basename(fileName).split(".")[0]; - const contentType = mime.lookup(path.extname(fileName)); + for (const fileName of fileNames) { + const blobName = path.basename(fileName).split(".")[0]; + const contentType = mime.lookup(path.extname(fileName)); - const blockBlobClient = containerClient.getBlockBlobClient(blobName); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); - await blockBlobClient.uploadFile(fileName, { - blobHTTPHeaders: { - blobContentType: contentType - } - }); + await blockBlobClient.uploadFile(fileName, { + blobHTTPHeaders: { + blobContentType: contentType + } + }); + } + } + catch (error) { + throw new Error(`Unable to upload media files. ${error.message}`); } } async function deleteBlobs(blobStorageUrl) { - const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); - const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); + try { + const blobServiceClient = new BlobServiceClient(blobStorageUrl.replace(`/${blobStorageContainer}`, "")); + const containerClient = blobServiceClient.getContainerClient(blobStorageContainer); - let blobs = containerClient.listBlobsFlat(); + let blobs = containerClient.listBlobsFlat(); - for await (const blob of blobs) { - const blockBlobClient = containerClient.getBlockBlobClient(blob.name); - await blockBlobClient.delete(); + for await (const blob of blobs) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete(); + } + } + catch (error) { + throw new Error(`Unable to delete media files. ${error.message}`); } } diff --git a/scripts.v3/cleanup.js b/scripts.v3/cleanup.js index f3b59651a..acd380ca1 100644 --- a/scripts.v3/cleanup.js +++ b/scripts.v3/cleanup.js @@ -58,7 +58,7 @@ cleanup() process.exit(0); }) .catch(error => { - console.error(error); + console.error(error.message); process.exit(1); });