Updates to new Azure Storage SDK

karma.conf.js

@@ -95,7 +95,6 @@ module.exports = function(config) {
                 { type: "cobertura", subdir: ".", file: "cobertura.xml" },
             ]
         },
-        // Can't enable yet has a conflict in dependency with azure-storage
         junitReporter: {
             outputDir: "./coverage"
         }

package.json

@@ -174,9 +174,8 @@
     "@angular/platform-server": "^11.0.0",
     "@angular/router": "^11.0.0",
     "@azure/msal-node": "^1.14.6",
-    "@azure/storage-blob": "^10.5.0",
+    "@azure/storage-blob": "^12.11.0",
     "applicationinsights": "^1.8.5",
-    "azure-storage": "^2.10.7",
     "chart.js": "^2.9.3",
     "chokidar": "^3.4.3",
     "commander": "^8.0.0",

scripts/azpipelines/update-latest.ts

@@ -1,75 +1,57 @@
 // eslint-disable no-console
-import * as azureStorage from "azure-storage";
 import makeDir from "make-dir";
 import * as path from "path";
 import * as fs from "fs";
-import { getManifest, getContainerName } from "./utils";
+import { getManifest, getContainerName, BlobStorageClient } from "./utils";
 import { promisify } from "util";
 
 const copyFile = promisify(fs.copyFile);
 
-const stagingDir = path.join(process.env.AGENT_TEMPDIRECTORY, "batchexplorer-github");
+if (!process.env.AGENT_TEMPDIRECTORY) {
+    throw new Error(
+        "Required AGENT_TEMPDIRECTORY environment variable is empty"
+    );
+}
+const stagingDir = path.join(process.env.AGENT_TEMPDIRECTORY,
+    "batchexplorer-github");
 console.log("Env", process.env);
 const storageAccountName = process.env.AZURE_STORAGE_ACCOUNT;
 const storageAccountKey = process.argv[2];
 
 console.log("Artifact staging directory is", stagingDir);
 console.log(`##vso[task.setvariable variable=BE_GITHUB_ARTIFACTS_DIR]${stagingDir}`)
 
+if (!storageAccountName) {
+    console.error(`No storage account name found in AZURE_STORAGE_ACCOUNT`);
+    process.exit(-1)
+}
+
 if (!storageAccountKey) {
     console.error("No storage account key passed");
     process.exit(-1);
 }
 
 console.log("Uploading to storage account:", storageAccountName);
-const blobService = azureStorage.createBlobService(storageAccountName, storageAccountKey);
-
-function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
 
-async function getProperties(container, blob): Promise<azureStorage.BlobService.BlobResult> {
-    return new Promise((resolve, reject) => {
-        blobService.getBlobProperties(container, blob, (error, result) => {
-            if (error) { return reject(error); }
-            resolve(result);
-        });
-    });
-}
+const storageClient = new BlobStorageClient(storageAccountName,
+    storageAccountKey);
 
-async function waitCopy(container, blob) {
-    while (true) {
-        const properties = await getProperties(container, blob);
-        const copyStatus = properties.copy && properties.copy.status;
-        switch (copyStatus) {
-            case "success":
-                return true;
-            case "aborted":
-                throw new Error("Copy was aborted");
-            case "failed":
-                throw new Error("Copy has failed");
-            case "pending":
-                console.log(`Copy "${blob}"is pending`);
+async function copyBlob(source, container, blob) {
+    const poller = await storageClient.beginCopyBlob(source, container, blob);
+    try {
+        return await poller.pollUntilDone();
+    } catch (error) {
+        switch (error.name) {
+            case "PollerCancelledError":
+                throw new Error(`Copy was cancelled: ${error.message}`);
+            case "PollerStoppedError":
+                throw new Error(`Copy was stopped: ${error.message}`);
+            default:
+                throw new Error(`Copy failed: ${error.message}`);
         }
-        sleep(5000);
     }
 }
 
-async function startCopyBlob(source, container, blob) {
-    return new Promise((resolve, reject) => {
-        blobService.startCopyBlob(source, container, blob, (error, result) => {
-            if (error) { return reject(error); }
-
-            resolve(result);
-        });
-    });
-}
-
-async function copyBlob(source, container, blob) {
-    await startCopyBlob(source, container, blob);
-    return waitCopy(container, blob);
-}
-
 function getLatestFile(os) {
     switch (os) {
         case "darwin":
@@ -83,9 +65,16 @@ function getLatestFile(os) {
 
 async function copyFilesToArtifactStaging(os) {
     const manifest = getManifest(os);
-    console.log(`Copy ${manifest.files.length} files for os: ${os}`);
-    for (const file of manifest.files) {
-        await copyFile(path.join(os, file.path), path.join(stagingDir, file.path))
+    if (manifest.files) {
+        console.log(`Copy ${manifest.files.length} files for os: ${os}`);
+        for (const file of manifest.files) {
+            if (file.path) {
+                await copyFile(
+                    path.join(os, file.path),
+                    path.join(stagingDir, file.path)
+                );
+            }
+        }
     }
 }
 
@@ -101,10 +90,15 @@ async function updateLatest(os) {
     const manifest = getManifest(os);
     console.log(`##vso[task.setvariable variable=BE_RELEASE_VERSION]${manifest.version}`)
     console.log(`Updating latest for os: ${os}`);
+    if (!manifest.buildType) {
+        throw new Error(
+            "Manifest does not contain required value for buildType"
+        );
+    }
     const container = getContainerName(manifest.buildType);
     const latestFile = getLatestFile(os);
-    const orgiginalBlob = `${manifest.version}/${latestFile}`;
-    const sourceUrl = blobService.getUrl(container, orgiginalBlob);
+    const originalBlob = `${manifest.version}/${latestFile}`;
+    const sourceUrl = storageClient.getUrl(container, originalBlob);
     console.log("Copying", sourceUrl, container, latestFile);
     return copyBlob(sourceUrl, container, latestFile);
 }

scripts/azpipelines/upload-to-storage.ts

@@ -1,8 +1,7 @@
 // eslint-disable no-console
 import * as fs from "fs";
 import * as path from "path";
-import * as AzureStorage from "azure-storage";
-import { getManifest, getContainerName } from "./utils";
+import { getManifest, getContainerName, BlobStorageClient } from "./utils";
 import * as crypto from "crypto";
 
 const storageAccountName = process.env.AZURE_STORAGE_ACCOUNT;
@@ -11,61 +10,43 @@ const attemptNumber = Number(process.env.Release_AttemptNumber);
 
 console.log(`This is the ${attemptNumber} try to release`);
 
+if (!storageAccountName) {
+    console.error(`No storage account name found in AZURE_STORAGE_ACCOUNT`);
+    process.exit(-1);
+}
+
 if (!storageAccountKey) {
     console.error("No storage account key passed");
     process.exit(-1);
 }
 
 console.log("Uploading to storage account:", storageAccountName);
-const blobService = AzureStorage.createBlobService(storageAccountName, storageAccountKey);
 
 function computeFileMd5(filename) {
     const data = fs.readFileSync(filename);
     return crypto.createHash("md5").update(data).digest("base64");
 }
 
-async function getBlob(container, blobName): Promise<AzureStorage.BlobService.BlobResult> {
-    return new Promise((resolve, reject) => {
-        blobService.getBlobProperties(container, blobName, (error, result) => {
-            if (error) {
-                return reject(error);
-            }
-
-            resolve(result);
-        });
-    });
-}
+const storageClient = new BlobStorageClient(storageAccountName,
+    storageAccountKey);
 
 async function createBlobFromLocalFile(container, filename, blobName, override = false) {
-    const options: AzureStorage.BlobService.CreateBlockBlobRequestOptions = {};
-    if (!override) {
-        options.accessConditions = AzureStorage.AccessCondition.generateIfNotExistsCondition();
-    }
-
-    return new Promise((resolve, reject) => {
-        blobService.createBlockBlobFromLocalFile(container, blobName, filename, options,
-            (error, result, response) => {
-
-                if (error) {
-                    reject(error);
-                    return;
-                }
+    const response = await storageClient.createBlob(container, blobName,
+        filename, override);
 
-                console.log("Uploaded", result, response);
-                resolve(result);
-            });
-    });
+    console.log("Uploaded", response);
+    return response;
 }
 
 async function uploadToBlob(container, filename, blobName, override = false) {
     console.log(`Uploading ${filename} ====> Container=${container}, Blobname=${blobName}`);
     try {
         return await createBlobFromLocalFile(container, filename, blobName, override);
     } catch (error) {
-        if (error.code === "BlobAlreadyExists") {
-            const blob = await getBlob(container, blobName);
+        if (error.details.errorCode === "BlobAlreadyExists") {
+            const blob = await storageClient.getBlob(container, blobName);
             const md5 = computeFileMd5(filename);
-            const blobMd5 = blob.contentSettings && blob.contentSettings.contentMD5;
+            const blobMd5 = blob.blobContentMD5?.toString();
             if (md5 === blobMd5) {
                 console.log(`Already uploaded ${filename} skipping(Md5 hash matched)`);
             } else {
@@ -79,10 +60,22 @@ async function uploadToBlob(container, filename, blobName, override = false) {
 
 async function uploadFiles(os) {
     const manifest = getManifest(os);
-    console.log(`Uploading ${manifest.files.length} files for os: ${os}`);
-    const container = getContainerName(manifest.buildType);
-    for (const file of manifest.files) {
-        await uploadToBlob(container, path.join(os, file.path), file.remotePath);
+    if (manifest.files) {
+        console.log(`Uploading ${manifest.files?.length} files for os: ${os}`);
+        if (!manifest.buildType) {
+            throw new Error(
+                "Manifest does not contain required value for buildType"
+            );
+        }
+        const container = getContainerName(manifest.buildType);
+        for (const file of manifest.files) {
+            if (file.path) {
+                await uploadToBlob(container, path.join(os, file.path),
+                    file.remotePath);
+            }
+        }
+    } else {
+        console.error(`Cannot get manifest for ${os}`);
     }
 }
 

scripts/azpipelines/utils.ts

@@ -1,3 +1,4 @@
+import { BlobBeginCopyFromURLResponse, BlobDownloadResponseParsed, BlobGetPropertiesResponse, BlobUploadCommonResponse, BlockBlobClient, PollerLike, PollOperationState, StorageSharedKeyCredential } from "@azure/storage-blob";
 import * as fs from "fs";
 import * as path from "path";
 
@@ -13,7 +14,11 @@ interface Manifest {
 }
 
 export function getManifest(os: string): Manifest {
-    return JSON.parse(fs.readFileSync(path.join(os, "manifest.json")).toString());
+    const filePath = path.join(os, "manifest.json");
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`Manifest path ${filePath} does not exist`);
+    }
+    return JSON.parse(fs.readFileSync(filePath).toString()) || {};
 }
 
 export function getContainerName(buildType: string): string {
@@ -26,3 +31,76 @@ export function getContainerName(buildType: string): string {
             return "test";
     }
 }
+
+
+export function storageURL(
+    account: string,
+    container?: string,
+    blob?: string
+): string {
+    let url = `https://${account}.blob.core.windows.net`;
+    if (container) {
+        url = `${url}/${container}`;
+        if (blob) {
+            url = `${url}/${blob}`;
+        }
+    }
+    return url;
+}
+
+export type CopyPoller = PollerLike<
+    PollOperationState<BlobBeginCopyFromURLResponse>,
+    BlobBeginCopyFromURLResponse
+>;
+export class BlobStorageClient {
+    private credential: StorageSharedKeyCredential;
+    constructor(private accountName: string, accountKey: string) {
+        this.credential = new StorageSharedKeyCredential(accountName,
+            accountKey);
+    }
+
+    private getBlobClient(container: string, blob: string) {
+        return new BlockBlobClient(
+            storageURL(this.accountName, container, blob),
+            this.credential
+        )
+    }
+
+    public getUrl(container: string, blob: string): string {
+        return storageURL(this.accountName, container, blob);
+    }
+
+    public async createBlob(
+        container: string,
+        blob: string,
+        filename: string,
+        override = true
+    ):
+        Promise<BlobUploadCommonResponse> {
+        const blobClient = this.getBlobClient(container, blob);
+        const conditions = override ? {} : { ifNoneMatch: "*" };
+        const response = await blobClient.uploadFile(filename, { conditions });
+        return response;
+    }
+
+    public async getBlob(container: string, blob: string):
+        Promise<BlobDownloadResponseParsed> {
+        return this.getBlobClient(container, blob).download(0);
+    }
+
+    public async getBlobProperties(container: string, blob: string):
+        Promise<BlobGetPropertiesResponse> {
+        return this.getBlobClient(container, blob).getProperties();
+    }
+
+    public async beginCopyBlob(source: string, container: string, blob: string):
+        Promise<CopyPoller> {
+        const client = this.getBlobClient(container, blob);
+        return client.beginCopyFromURL(source, {
+            intervalInMs: 5000,
+            onProgress(state) {
+                console.log(`Copy "${blob}" is pending ${state.copyProgress}`);
+            },
+        })
+    }
+}