From 6205e1dc4facd4d41cd351c8c2ec758a8343d1f9 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 10:59:29 -0700 Subject: [PATCH 01/64] use async await --- src/CSharpExtDownloader.ts | 16 +++++- src/omnisharp/OmnisharpDownloader.ts | 8 +-- src/packages.ts | 77 +++++++++++----------------- 3 files changed, 49 insertions(+), 52 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 6d3ee4a57..b875ab8eb 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -5,7 +5,7 @@ import * as util from './common'; import { GetNetworkConfiguration } from './downloader.helper'; -import { PackageManager } from './packages'; +import { PackageManager, Package } from './packages'; import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; @@ -28,15 +28,19 @@ export class CSharpExtDownloader { try { await util.touchInstallFile(util.InstallFileType.Begin); - let packageManager = new PackageManager(this.platformInfo, this.packageJSON); + let packageManager = new PackageManager(this.platformInfo); // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); installationStage = 'downloadPackages'; + // shoule we move this to the omnisharp manager and then unify the downloaders + let packages = this.GetRunTimeDependenciesPackages(); let networkConfiguration = GetNetworkConfiguration(); const proxy = networkConfiguration.Proxy; const strictSSL = networkConfiguration.StrictSSL; + packageManager.SetPackagesToDownload(packages); + await packageManager.DownloadPackages(this.eventStream, proxy, strictSSL); installationStage = 'installPackages'; @@ -61,4 +65,12 @@ export class CSharpExtDownloader { return success; } } + + public GetRunTimeDependenciesPackages(): Package[] { + if (this.packageJSON.runtimeDependencies) { + return JSON.parse(JSON.stringify(this.packageJSON.runtimeDependencies)); + } + + return null; + } } diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 25fe81141..bf4889be1 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -10,9 +10,9 @@ import { PlatformInformation } from '../platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; import { EventStream } from '../EventStream'; -const defaultPackageManagerFactory: IPackageManagerFactory = (platformInfo, packageJSON) => new PackageManager(platformInfo, packageJSON); +const defaultPackageManagerFactory: IPackageManagerFactory = (platformInfo) => new PackageManager(platformInfo); export interface IPackageManagerFactory { - (platformInfo: PlatformInformation, packageJSON: any): PackageManager; + (platformInfo: PlatformInformation): PackageManager; } export class OmnisharpDownloader { @@ -29,7 +29,7 @@ export class OmnisharpDownloader { let networkConfiguration = GetNetworkConfiguration(); this.proxy = networkConfiguration.Proxy; this.strictSSL = networkConfiguration.StrictSSL; - this.packageManager = packageManagerFactory(this.platformInfo, this.packageJSON); + this.packageManager = packageManagerFactory(this.platformInfo); } public async DownloadAndInstallOmnisharp(version: string, serverUrl: string, installPath: string) { @@ -44,7 +44,7 @@ export class OmnisharpDownloader { installationStage = 'downloadPackages'; // Specify the packages that the package manager needs to download - this.packageManager.SetVersionPackagesForDownload(packages); + this.packageManager.SetPackagesToDownload(packages); await this.packageManager.DownloadPackages(this.eventStream, this.proxy, this.strictSSL); installationStage = 'installPackages'; diff --git a/src/packages.ts b/src/packages.ts index 210a9a32f..2b2fe2005 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -44,25 +44,20 @@ export class PackageManager { private allPackages: Package[]; public constructor( - private platformInfo: PlatformInformation, - private packageJSON: any) { + private platformInfo: PlatformInformation) { // Ensure our temp files get cleaned up in case of error. tmp.setGracefulCleanup(); } public async DownloadPackages(eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - return this.GetPackages() - .then(async packages => { - return util.buildPromiseChain(packages, async pkg => maybeDownloadPackage(pkg, eventStream, proxy, strictSSL)); - }); + let packages = await this.GetPackages(); + return util.buildPromiseChain(packages, async pkg => maybeDownloadPackage(pkg, eventStream, proxy, strictSSL)); } public async InstallPackages(eventStream: EventStream): Promise { - return this.GetPackages() - .then(async packages => { - return util.buildPromiseChain(packages, async pkg => installPackage(pkg, eventStream)); - }); + let packages = await this.GetPackages(); + return util.buildPromiseChain(packages, async pkg => installPackage(pkg, eventStream)); } private async GetAllPackages(): Promise { @@ -70,15 +65,6 @@ export class PackageManager { if (this.allPackages) { resolve(this.allPackages); } - else if (this.packageJSON.runtimeDependencies) { - this.allPackages = JSON.parse(JSON.stringify(this.packageJSON.runtimeDependencies)); - //Copying the packages by value and not by reference so that there are no side effects - - // Convert relative binary paths to absolute - resolvePackageBinaries(this.allPackages); - - resolve(this.allPackages); - } else { reject(new PackageError("Package manifest does not exist.")); } @@ -86,23 +72,21 @@ export class PackageManager { } private async GetPackages(): Promise { - return this.GetAllPackages() - .then(list => { - return list.filter(pkg => { - if (pkg.architectures && pkg.architectures.indexOf(this.platformInfo.architecture) === -1) { - return false; - } - - if (pkg.platforms && pkg.platforms.indexOf(this.platformInfo.platform) === -1) { - return false; - } - - return true; - }); - }); + let list = await this.GetAllPackages(); + return list.filter(pkg => { + if (pkg.architectures && pkg.architectures.indexOf(this.platformInfo.architecture) === -1) { + return false; + } + + if (pkg.platforms && pkg.platforms.indexOf(this.platformInfo.platform) === -1) { + return false; + } + + return true; + }); } - public SetVersionPackagesForDownload(packages: Package[]) { + public SetPackagesToDownload(packages: Package[]) { this.allPackages = packages; resolvePackageBinaries(this.allPackages); } @@ -127,11 +111,13 @@ export class PackageManager { function resolvePackageBinaries(packages: Package[]) { // Convert relative binary paths to absolute - for (let pkg of packages) { - if (pkg.binaries) { - pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); + if (packages) { + for (let pkg of packages) { + if (pkg.binaries) { + pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); + } } - } + } } function getBaseInstallPath(pkg: Package): string { @@ -144,13 +130,12 @@ function getBaseInstallPath(pkg: Package): string { } async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - return doesPackageTestPathExist(pkg).then(async (exists: boolean) => { - if (!exists) { - return downloadPackage(pkg, eventStream, proxy, strictSSL); - } else { - eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); - } - }); + let exists = doesPackageTestPathExist(pkg); + if (!exists) { + return downloadPackage(pkg, eventStream, proxy, strictSSL); + } else { + eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); + } } async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { @@ -178,7 +163,7 @@ async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: st result = result.catch(async (primaryUrlError) => { eventStream.post(new DownloadFallBack(pkg.fallbackUrl)); return downloadFile(pkg.fallbackUrl, pkg, eventStream, proxy, strictSSL) - .then(() => eventStream.post(new DownloadSuccess(' Done!' ))) + .then(() => eventStream.post(new DownloadSuccess(' Done!'))) .catch(() => primaryUrlError); }); } From c632ab261081efbd7a57fb0f80b86695de95ccdf Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 12:17:18 -0700 Subject: [PATCH 02/64] async await --- src/packages.ts | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/packages.ts b/src/packages.ts index 2b2fe2005..d413d6be3 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -61,6 +61,13 @@ export class PackageManager { } private async GetAllPackages(): Promise { + if (this.allPackages) { + return this.allPackages; + } + else { + throw new PackageError("Package manifest does not exist."); + } + return new Promise((resolve, reject) => { if (this.allPackages) { resolve(this.allPackages); @@ -117,7 +124,7 @@ function resolvePackageBinaries(packages: Package[]) { pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); } } - } + } } function getBaseInstallPath(pkg: Package): string { @@ -139,10 +146,8 @@ async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, prox } async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - eventStream.post(new DownloadStart(pkg.description)); - - return new Promise((resolve, reject) => { + const tmpResult = await new Promise((resolve, reject) => { tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { if (err) { return reject(new PackageError('Error from tmp.file', pkg, err)); @@ -150,26 +155,32 @@ async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: st resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); }); - }).then(async tmpResult => { - pkg.tmpFile = tmpResult; + }); - let result = downloadFile(pkg.url, pkg, eventStream, proxy, strictSSL) - .then(() => eventStream.post(new DownloadSuccess(` Done!`))); + pkg.tmpFile = tmpResult; + try { + await downloadFile(pkg.url, pkg, eventStream, proxy, strictSSL); + eventStream.post(new DownloadSuccess(` Done!`)); + } + catch (primaryUrlError) { // If the package has a fallback Url, and downloading from the primary Url failed, try again from // the fallback. This is used for debugger packages as some users have had issues downloading from - // the CDN link. + // the CDN link if (pkg.fallbackUrl) { - result = result.catch(async (primaryUrlError) => { - eventStream.post(new DownloadFallBack(pkg.fallbackUrl)); - return downloadFile(pkg.fallbackUrl, pkg, eventStream, proxy, strictSSL) - .then(() => eventStream.post(new DownloadSuccess(' Done!'))) - .catch(() => primaryUrlError); - }); + eventStream.post(new DownloadFallBack(pkg.fallbackUrl)); + try { + await downloadFile(pkg.fallbackUrl, pkg, eventStream, proxy, strictSSL); + eventStream.post(new DownloadSuccess(' Done!')); + } + catch (fallbackUrlError) { + throw primaryUrlError; + } } - - return result; - }); + else { + throw primaryUrlError; + } + } } async function downloadFile(urlString: string, pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { @@ -179,6 +190,7 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS host: url.host, path: url.path, agent: getProxyAgent(url, proxy, strictSSL), + port : url.port, rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true }; From 01f8cb1f28a079204d8ae366003e9cba2628a069 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 12:33:17 -0700 Subject: [PATCH 03/64] remove promise --- src/packages.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/packages.ts b/src/packages.ts index d413d6be3..31cf08aa0 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -67,15 +67,6 @@ export class PackageManager { else { throw new PackageError("Package manifest does not exist."); } - - return new Promise((resolve, reject) => { - if (this.allPackages) { - resolve(this.allPackages); - } - else { - reject(new PackageError("Package manifest does not exist.")); - } - }); } private async GetPackages(): Promise { From 56675b9def1b14ffac457e6f6df8fb33c751f291 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 15:11:45 -0700 Subject: [PATCH 04/64] remove packagejson --- src/CSharpExtDownloader.ts | 6 +-- src/omnisharp/OmnisharpDownloader.ts | 9 ++-- src/packages.ts | 69 ++++++++++++++-------------- tasks/offlinePackagingTasks.ts | 2 +- 4 files changed, 41 insertions(+), 45 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index b875ab8eb..78025e664 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -39,12 +39,10 @@ export class CSharpExtDownloader { const proxy = networkConfiguration.Proxy; const strictSSL = networkConfiguration.StrictSSL; - packageManager.SetPackagesToDownload(packages); - - await packageManager.DownloadPackages(this.eventStream, proxy, strictSSL); + let downloadedPackages = await packageManager.DownloadPackages(packages, this.eventStream, proxy, strictSSL); installationStage = 'installPackages'; - await packageManager.InstallPackages(this.eventStream); + await packageManager.InstallPackages(downloadedPackages, this.eventStream); installationStage = 'touchLockFile'; await util.touchInstallFile(util.InstallFileType.Lock); diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index bf4889be1..a71d14d2d 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -40,15 +40,14 @@ export class OmnisharpDownloader { this.eventStream.post(new LogPlatformInfo(this.platformInfo)); installationStage = 'getPackageInfo'; - let packages: Package[] = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); + let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); installationStage = 'downloadPackages'; - // Specify the packages that the package manager needs to download - this.packageManager.SetPackagesToDownload(packages); - await this.packageManager.DownloadPackages(this.eventStream, this.proxy, this.strictSSL); + + let downloadedPackages = await this.packageManager.DownloadPackages(packages, this.eventStream, this.proxy, this.strictSSL); installationStage = 'installPackages'; - await this.packageManager.InstallPackages(this.eventStream); + await this.packageManager.InstallPackages(downloadedPackages, this.eventStream); this.eventStream.post(new InstallationSuccess()); } diff --git a/src/packages.ts b/src/packages.ts index 31cf08aa0..3dd349f13 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -41,52 +41,48 @@ export class PackageError extends Error { } export class PackageManager { - private allPackages: Package[]; - public constructor( private platformInfo: PlatformInformation) { - // Ensure our temp files get cleaned up in case of error. tmp.setGracefulCleanup(); } - public async DownloadPackages(eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let packages = await this.GetPackages(); - return util.buildPromiseChain(packages, async pkg => maybeDownloadPackage(pkg, eventStream, proxy, strictSSL)); + public async DownloadPackages(packages: Package[], eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { + let packagesToDownload = await this.GetPackages(packages); + for(let pkg of packagesToDownload){ + await maybeDownloadPackage(pkg, eventStream, proxy, strictSSL); + } + return packagesToDownload; } - public async InstallPackages(eventStream: EventStream): Promise { - let packages = await this.GetPackages(); - return util.buildPromiseChain(packages, async pkg => installPackage(pkg, eventStream)); + public async InstallPackages(packages: Package[], eventStream: EventStream): Promise { + let packagesToInstall = await this.GetPackages(packages); + return util.buildPromiseChain(packagesToInstall, async pkg => installPackage(pkg, eventStream)); } - private async GetAllPackages(): Promise { - if (this.allPackages) { - return this.allPackages; - } - else { - throw new PackageError("Package manifest does not exist."); - } + public async GetPackages(packages: Package[]) { + let filteredPackages = await this.FilterPackages(packages); + resolvePackageBinaries(filteredPackages); + return filteredPackages; } - private async GetPackages(): Promise { - let list = await this.GetAllPackages(); - return list.filter(pkg => { - if (pkg.architectures && pkg.architectures.indexOf(this.platformInfo.architecture) === -1) { - return false; - } - - if (pkg.platforms && pkg.platforms.indexOf(this.platformInfo.platform) === -1) { - return false; - } + private async FilterPackages(packages: Package[]) { + if (packages) { + return packages.filter(pkg => { + if (pkg.architectures && pkg.architectures.indexOf(this.platformInfo.architecture) === -1) { + return false; + } - return true; - }); - } + if (pkg.platforms && pkg.platforms.indexOf(this.platformInfo.platform) === -1) { + return false; + } - public SetPackagesToDownload(packages: Package[]) { - this.allPackages = packages; - resolvePackageBinaries(this.allPackages); + return true; + }); + } + else { + throw new PackageError("Package manifest does not exist."); + } } public async GetLatestVersionFromFile(eventStream: EventStream, proxy: string, strictSSL: boolean, filePackage: Package): Promise { @@ -127,16 +123,17 @@ function getBaseInstallPath(pkg: Package): string { return basePath; } -async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { +async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { let exists = doesPackageTestPathExist(pkg); if (!exists) { return downloadPackage(pkg, eventStream, proxy, strictSSL); } else { eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); + return pkg; } } -async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { +async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { eventStream.post(new DownloadStart(pkg.description)); const tmpResult = await new Promise((resolve, reject) => { tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { @@ -153,6 +150,7 @@ async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: st try { await downloadFile(pkg.url, pkg, eventStream, proxy, strictSSL); eventStream.post(new DownloadSuccess(` Done!`)); + return pkg; //return the downloaded package } catch (primaryUrlError) { // If the package has a fallback Url, and downloading from the primary Url failed, try again from @@ -163,6 +161,7 @@ async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: st try { await downloadFile(pkg.fallbackUrl, pkg, eventStream, proxy, strictSSL); eventStream.post(new DownloadSuccess(' Done!')); + return pkg; } catch (fallbackUrlError) { throw primaryUrlError; @@ -181,7 +180,7 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS host: url.host, path: url.path, agent: getProxyAgent(url, proxy, strictSSL), - port : url.port, + port: url.port, rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true }; diff --git a/tasks/offlinePackagingTasks.ts b/tasks/offlinePackagingTasks.ts index 550e998bc..8418e0f4a 100644 --- a/tasks/offlinePackagingTasks.ts +++ b/tasks/offlinePackagingTasks.ts @@ -87,7 +87,7 @@ async function doOfflinePackage(platformInfo: PlatformInformation, packageName: // Install Tasks async function install(platformInfo: PlatformInformation, packageJSON: any) { - const packageManager = new PackageManager(platformInfo, packageJSON); + const packageManager = new PackageManager(platformInfo); let eventStream = new EventStream(); const logger = new Logger(message => process.stdout.write(message)); let stdoutObserver = new CsharpLoggerObserver(logger); From 400ba5ec2aef1607a4569fa7c2c8dc2a94143709 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 10 Apr 2018 15:49:55 -0700 Subject: [PATCH 05/64] Feature tests running with refactored package manager --- src/CSharpExtDownloader.ts | 6 +++--- src/omnisharp/OmnisharpDownloader.ts | 2 +- src/packages.ts | 2 +- tasks/offlinePackagingTasks.ts | 7 ++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 8d2207022..6f6a2bd11 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -5,7 +5,7 @@ import * as util from './common'; import { GetNetworkConfiguration, defaultPackageManagerFactory, IPackageManagerFactory } from './downloader.helper'; -import { PackageManager } from './packages'; +import { PackageManager, Package } from './packages'; import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; @@ -43,9 +43,9 @@ export class CSharpExtDownloader { // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); - let runTimeDependencies = this.GetRunTimeDependenciesPackages(); + let runTimeDependencies = this.GetRunTimeDependenciesPackages(); + installationStage = 'downloadPackages'; - let downloadedPackages = await this.packageManager.DownloadPackages(runTimeDependencies, this.eventStream, this.proxy, this.strictSSL); installationStage = 'installPackages'; diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index a408af87c..15c331668 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -5,7 +5,7 @@ import { GetNetworkConfiguration, IPackageManagerFactory, defaultPackageManagerFactory } from '../downloader.helper'; import { GetPackagesFromVersion, GetVersionFilePackage } from './OmnisharpPackageCreator'; -import { Package, PackageManager } from '../packages'; +import { PackageManager } from '../packages'; import { PlatformInformation } from '../platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; import { EventStream } from '../EventStream'; diff --git a/src/packages.ts b/src/packages.ts index 3dd349f13..659cafff8 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -124,7 +124,7 @@ function getBaseInstallPath(pkg: Package): string { } async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let exists = doesPackageTestPathExist(pkg); + let exists = await doesPackageTestPathExist(pkg); if (!exists) { return downloadPackage(pkg, eventStream, proxy, strictSSL); } else { diff --git a/tasks/offlinePackagingTasks.ts b/tasks/offlinePackagingTasks.ts index 8418e0f4a..3fa782b5a 100644 --- a/tasks/offlinePackagingTasks.ts +++ b/tasks/offlinePackagingTasks.ts @@ -19,7 +19,7 @@ import { CsharpLoggerObserver } from '../src/observers/CsharpLoggerObserver'; import { EventStream } from '../src/EventStream'; import { getPackageJSON } from '../tasks/packageJson'; import { Logger } from '../src/logger'; -import { PackageManager } from '../src/packages'; +import { PackageManager, Package } from '../src/packages'; import { PlatformInformation } from '../src/platform'; gulp.task('vsix:offline:package', async () => { @@ -93,9 +93,10 @@ async function install(platformInfo: PlatformInformation, packageJSON: any) { let stdoutObserver = new CsharpLoggerObserver(logger); eventStream.subscribe(stdoutObserver.post); const debuggerUtil = new debugUtil.CoreClrDebugUtil(path.resolve('.')); + let runTimeDependencies = JSON.parse(JSON.stringify(packageJSON.runtimeDependencies)); - await packageManager.DownloadPackages(eventStream, undefined, undefined); - await packageManager.InstallPackages(eventStream); + let downloadedPackages = await packageManager.DownloadPackages(runTimeDependencies, eventStream, undefined, undefined); + await packageManager.InstallPackages(downloadedPackages, eventStream); await util.touchInstallFile(util.InstallFileType.Lock); await debugUtil.CoreClrDebugUtil.writeEmptyFile(debuggerUtil.installCompleteFilePath()); } From 5c4bfc049bd2457114270d76fa781688e681847c Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 17:15:52 -0700 Subject: [PATCH 06/64] trial to mock server --- src/packages.ts | 1 + test/unitTests/packageManager.test.ts | 69 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 test/unitTests/packageManager.test.ts diff --git a/src/packages.ts b/src/packages.ts index 659cafff8..a0615d485 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -180,6 +180,7 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS host: url.host, path: url.path, agent: getProxyAgent(url, proxy, strictSSL), + //look into this port: url.port, rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true }; diff --git a/test/unitTests/packageManager.test.ts b/test/unitTests/packageManager.test.ts new file mode 100644 index 000000000..5717eedce --- /dev/null +++ b/test/unitTests/packageManager.test.ts @@ -0,0 +1,69 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as tmp from 'tmp'; +import * as util from '../../src/common'; +import { rimraf } from 'async-file'; +import { PlatformInformation } from '../../src/platform'; +import { EventStream } from '../../src/EventStream'; +import { PackageManager, Package } from '../../src/packages'; +let ServerMock = require("mock-http-server"); +const chai = require("chai"); +chai.use(require("chai-as-promised")); +let expect = chai.expect; + +suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downloads and installs them", () => { + let server = new ServerMock({ host: "127.0.0.1", port: 9000 }); + let tmpDir: tmp.SynchrounousResult = null; + const platformInfo = new PlatformInformation("win32", "x86"); + const eventStream = new EventStream(); + const manager = new PackageManager(platformInfo); + //const serverUrl = "https://roslynomnisharp.blob.core.windows.net"; + const serverUrl = "http://127.0.0.1:9000"; + + setup(() => { + tmpDir = tmp.dirSync(); + util.setExtensionPath(tmpDir.name); + }); + + test('Packages are downloaded from the specified server url and installed at the specified path', function(done){ + /* Download a test package that conatins a install_check_1.2.3.txt file and check whether the + file appears at the expected path */ + + server.on({ + method: 'GET', + path: '/resource', + reply: { + status: 200, + headers: { "content-type": "text/plain" }, + body: "1.28.0" + } + }); + let packages = Array( { + "description": "Latest version information file", + "url": `${serverUrl}/resource` + }); + console.log(packages[0].url); + + try { + manager.DownloadPackages(packages, eventStream, undefined, undefined); + } + catch (error) { + console.log(error); + } + + done(); + //expect(downloadedPackage).to.not.be.empty; + //expect(downloadedPackage[0].tmpFile).to.not.be.undefined; + }); + + teardown(async () => { + if (tmpDir) { + await rimraf(tmpDir.name); + } + + tmpDir = null; + }); +}); From 396890070c9af518e7f22e086c5200bca829d8b9 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 10 Apr 2018 17:16:17 -0700 Subject: [PATCH 07/64] package add --- package-lock.json | 248 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 1 + 2 files changed, 248 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 80daf6260..fad7855da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -695,6 +695,32 @@ "inherits": "2.0.3" } }, + "body-parser": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", + "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "dev": true, + "requires": { + "bytes": "3.0.0", + "content-type": "1.0.4", + "debug": "2.6.9", + "depd": "1.1.2", + "http-errors": "1.6.3", + "iconv-lite": "0.4.19", + "on-finished": "2.3.0", + "qs": "6.5.1", + "raw-body": "2.3.2", + "type-is": "1.6.16" + }, + "dependencies": { + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", + "dev": true + } + } + }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -754,6 +780,12 @@ "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", "dev": true }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=", + "dev": true + }, "call-me-maybe": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz", @@ -1106,6 +1138,24 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "connect": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/connect/-/connect-3.6.6.tgz", + "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=", + "dev": true, + "requires": { + "debug": "2.6.9", + "finalhandler": "1.1.0", + "parseurl": "1.3.2", + "utils-merge": "1.0.1" + } + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "dev": true + }, "convert-source-map": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.5.1.tgz", @@ -1403,6 +1453,12 @@ "integrity": "sha1-OjYof1A05pnnV3kBBSwubJQlFjE=", "dev": true }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "dev": true + }, "deprecated": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/deprecated/-/deprecated-0.0.1.tgz", @@ -1564,6 +1620,18 @@ "jsbn": "0.1.1" } }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=", + "dev": true + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "dev": true + }, "end-of-stream": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-0.1.5.tgz", @@ -1636,6 +1704,12 @@ "es6-promise": "4.1.1" } }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "dev": true + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -1687,7 +1761,7 @@ }, "event-stream": { "version": "3.3.4", - "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "resolved": "http://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=", "dev": true, "requires": { @@ -1886,6 +1960,29 @@ "repeat-string": "1.6.1" } }, + "finalhandler": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz", + "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", + "dev": true, + "requires": { + "debug": "2.6.9", + "encodeurl": "1.0.2", + "escape-html": "1.0.3", + "on-finished": "2.3.0", + "parseurl": "1.3.2", + "statuses": "1.3.1", + "unpipe": "1.0.0" + }, + "dependencies": { + "statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=", + "dev": true + } + } + }, "find-index": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/find-index/-/find-index-0.1.1.tgz", @@ -3455,6 +3552,18 @@ } } }, + "http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "dev": true, + "requires": { + "depd": "1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": "1.5.0" + } + }, "http-proxy-agent": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.0.0.tgz", @@ -3494,6 +3603,12 @@ } } }, + "iconv-lite": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==", + "dev": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -4515,6 +4630,12 @@ "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=", "dev": true }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "dev": true + }, "mem": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", @@ -4722,6 +4843,18 @@ "yargs": "6.6.0" } }, + "mock-http-server": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/mock-http-server/-/mock-http-server-0.2.0.tgz", + "integrity": "sha1-7e/sDoC+i8oobx4MAN6Nv+vw7JA=", + "dev": true, + "requires": { + "body-parser": "1.18.2", + "connect": "3.6.6", + "multiparty": "4.1.3", + "underscore": "1.8.3" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -4739,6 +4872,15 @@ "minimatch": "3.0.4" } }, + "multiparty": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/multiparty/-/multiparty-4.1.3.tgz", + "integrity": "sha1-PEPH/LGJbhdGBDap3Qtu8WaOT5Q=", + "dev": true, + "requires": { + "fd-slicer": "1.0.1" + } + }, "multipipe": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/multipipe/-/multipipe-0.1.2.tgz", @@ -7746,6 +7888,15 @@ } } }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dev": true, + "requires": { + "ee-first": "1.1.1" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -7943,6 +8094,12 @@ "@types/node": "9.6.1" } }, + "parseurl": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=", + "dev": true + }, "path-dirname": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", @@ -8234,6 +8391,44 @@ } } }, + "raw-body": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", + "dev": true, + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.2", + "iconv-lite": "0.4.19", + "unpipe": "1.0.0" + }, + "dependencies": { + "depd": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=", + "dev": true + }, + "http-errors": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", + "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", + "dev": true, + "requires": { + "depd": "1.1.1", + "inherits": "2.0.3", + "setprototypeof": "1.0.3", + "statuses": "1.5.0" + } + }, + "setprototypeof": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", + "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=", + "dev": true + } + } + }, "read": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", @@ -8732,6 +8927,12 @@ "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", "dev": true }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -8907,6 +9108,12 @@ "integrity": "sha1-5sgLYjEj19gM8TLOU480YokHJQI=", "dev": true }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "dev": true + }, "stream-combiner": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", @@ -9403,6 +9610,33 @@ "integrity": "sha512-N9IvkQslUGYGC24RkJk1ba99foK6TkwC2FHAEBlQFBP0RxQZS8ZpJuAZcwiY/w9ZJHFQb1aOXBI60OdxhTrwEQ==", "dev": true }, + "type-is": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "dev": true, + "requires": { + "media-typer": "0.3.0", + "mime-types": "2.1.18" + }, + "dependencies": { + "mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "dev": true + }, + "mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "dev": true, + "requires": { + "mime-db": "1.33.0" + } + } + } + }, "typed-rest-client": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-0.9.0.tgz", @@ -9515,6 +9749,12 @@ "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=" }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "dev": true + }, "unzip2": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/unzip2/-/unzip2-0.2.5.tgz", @@ -9598,6 +9838,12 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "dev": true + }, "uuid": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", diff --git a/package.json b/package.json index 87cb1e564..3e368cc0f 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "minimist": "^1.2.0", "mocha": "^5.0.4", "mocha-typescript": "^1.1.12", + "mock-http-server": "^0.2.0", "npm-run-all": "^4.1.2", "nyc": "^11.6.0", "plist": "^3.0.1", From cbf50af5180d956b169ceb149cb6164d5d0ada36 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 11 Apr 2018 11:45:11 -0700 Subject: [PATCH 08/64] package manager test not running --- src/packages.ts | 3 ++- test/unitTests/packageManager.test.ts | 32 +++++++++++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/packages.ts b/src/packages.ts index a0615d485..7a47b15af 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -177,7 +177,7 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS const url = parseUrl(urlString); const options: https.RequestOptions = { - host: url.host, + host: "127.0.0.1", path: url.path, agent: getProxyAgent(url, proxy, strictSSL), //look into this @@ -234,6 +234,7 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS }); request.on('error', err => { + console.log(err); reject(new PackageError(`Request error: ${err.message || 'NONE'}`, pkg, err)); }); diff --git a/test/unitTests/packageManager.test.ts b/test/unitTests/packageManager.test.ts index 5717eedce..b6f39bec5 100644 --- a/test/unitTests/packageManager.test.ts +++ b/test/unitTests/packageManager.test.ts @@ -12,7 +12,7 @@ import { PackageManager, Package } from '../../src/packages'; let ServerMock = require("mock-http-server"); const chai = require("chai"); chai.use(require("chai-as-promised")); -let expect = chai.expect; +//let expect = chai.expect; suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downloads and installs them", () => { let server = new ServerMock({ host: "127.0.0.1", port: 9000 }); @@ -21,40 +21,48 @@ suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downlo const eventStream = new EventStream(); const manager = new PackageManager(platformInfo); //const serverUrl = "https://roslynomnisharp.blob.core.windows.net"; - const serverUrl = "http://127.0.0.1:9000"; + const serverUrl = "https://127.0.0.1:9000"; - setup(() => { + setup(function (done) { + server.start(done); tmpDir = tmp.dirSync(); util.setExtensionPath(tmpDir.name); }); - test('Packages are downloaded from the specified server url and installed at the specified path', function(done){ + /*teardown(function (done) { + server.stop(done); + });*/ + + test('Packages are downloaded from the specified server url and installed at the specified path', function (done) { /* Download a test package that conatins a install_check_1.2.3.txt file and check whether the file appears at the expected path */ - - server.on({ + + server.on({ method: 'GET', path: '/resource', reply: { - status: 200, + status: 200, headers: { "content-type": "text/plain" }, - body: "1.28.0" + body: "1.28.0" } }); - let packages = Array( { + let packages = Array({ "description": "Latest version information file", "url": `${serverUrl}/resource` }); console.log(packages[0].url); - + try { - manager.DownloadPackages(packages, eventStream, undefined, undefined); + manager.DownloadPackages(packages, eventStream, undefined, undefined).then(() => { + done(); + }); } catch (error) { console.log(error); + done(); } - done(); + //done(); //expect(downloadedPackage).to.not.be.empty; //expect(downloadedPackage[0].tmpFile).to.not.be.undefined; }); From 8c09369e71974b300fda5fc2c1dc33b9820b47bf Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 11 Apr 2018 16:34:43 -0700 Subject: [PATCH 09/64] Mocking the server running --- .vscode/launch.json | 17 ++++++++++++ src/packages.ts | 3 +-- test/unitTests/packageManager.test.ts | 38 +++++++++++++++------------ test/unitTests/testAssets/private.pem | 27 +++++++++++++++++++ test/unitTests/testAssets/public.pem | 21 +++++++++++++++ 5 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 test/unitTests/testAssets/private.pem create mode 100644 test/unitTests/testAssets/public.pem diff --git a/.vscode/launch.json b/.vscode/launch.json index 09c6d35ef..0c0cf168e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,23 @@ { "version": "0.2.0", "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Unit tests", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/out/test/unitTests/**/*.test.js" + ], + "sourceMaps": true, + "internalConsoleOptions": "openOnSessionStart", + "outFiles" :["${workspaceRoot}/out/test/**/*.js"] + }, { "name": "Launch Extension", "type": "extensionHost", diff --git a/src/packages.ts b/src/packages.ts index 7a47b15af..520f8759a 100644 --- a/src/packages.ts +++ b/src/packages.ts @@ -177,10 +177,9 @@ async function downloadFile(urlString: string, pkg: Package, eventStream: EventS const url = parseUrl(urlString); const options: https.RequestOptions = { - host: "127.0.0.1", + host: url.hostname, path: url.path, agent: getProxyAgent(url, proxy, strictSSL), - //look into this port: url.port, rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true }; diff --git a/test/unitTests/packageManager.test.ts b/test/unitTests/packageManager.test.ts index b6f39bec5..6d07e16f0 100644 --- a/test/unitTests/packageManager.test.ts +++ b/test/unitTests/packageManager.test.ts @@ -9,19 +9,27 @@ import { rimraf } from 'async-file'; import { PlatformInformation } from '../../src/platform'; import { EventStream } from '../../src/EventStream'; import { PackageManager, Package } from '../../src/packages'; +import * as fs from 'fs'; + let ServerMock = require("mock-http-server"); const chai = require("chai"); chai.use(require("chai-as-promised")); -//let expect = chai.expect; +let expect = chai.expect; suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downloads and installs them", () => { - let server = new ServerMock({ host: "127.0.0.1", port: 9000 }); + let server = new ServerMock({ host: "127.0.0.1", port: 9001 }, + { + host: "localhost", + port: 9002, + key: fs.readFileSync("test/unitTests/testAssets/private.pem"), + cert: fs.readFileSync("test/unitTests/testAssets/public.pem") + }); let tmpDir: tmp.SynchrounousResult = null; const platformInfo = new PlatformInformation("win32", "x86"); const eventStream = new EventStream(); const manager = new PackageManager(platformInfo); //const serverUrl = "https://roslynomnisharp.blob.core.windows.net"; - const serverUrl = "https://127.0.0.1:9000"; + const serverUrl = "https://127.0.0.1:9002"; setup(function (done) { server.start(done); @@ -29,42 +37,38 @@ suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downlo util.setExtensionPath(tmpDir.name); }); - /*teardown(function (done) { + teardown(function (done) { server.stop(done); - });*/ - - test('Packages are downloaded from the specified server url and installed at the specified path', function (done) { - /* Download a test package that conatins a install_check_1.2.3.txt file and check whether the - file appears at the expected path */ + }); + test('Packages are downloaded from the specified server url and installed at the specified path', async() =>{ + console.log("hello1"); server.on({ method: 'GET', path: '/resource', reply: { status: 200, headers: { "content-type": "text/plain" }, - body: "1.28.0" + body: "something" } }); + let packages = Array({ "description": "Latest version information file", "url": `${serverUrl}/resource` }); console.log(packages[0].url); + let downloadedPackage : Package[]; try { - manager.DownloadPackages(packages, eventStream, undefined, undefined).then(() => { - done(); - }); + downloadedPackage = await manager.DownloadPackages(packages, eventStream, undefined, false); } catch (error) { console.log(error); - done(); } - //done(); - //expect(downloadedPackage).to.not.be.empty; - //expect(downloadedPackage[0].tmpFile).to.not.be.undefined; + expect(downloadedPackage).to.not.be.empty; + expect(downloadedPackage[0].tmpFile).to.not.be.undefined; }); teardown(async () => { diff --git a/test/unitTests/testAssets/private.pem b/test/unitTests/testAssets/private.pem new file mode 100644 index 000000000..9b11477cc --- /dev/null +++ b/test/unitTests/testAssets/private.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAu1cmHrPygWBVWr0ng17mEXxW+jgre5vi39GLDXbSczM6rFJ7 +M0Um/4UhSWttcnP48ctg0osm4cQRrgC/F05MCQyPWFmq0/SV+Ct86KM1jRpW0nlw +fqX7kxZfA6aKR3XJb/PNhX/LrKMCj3hTlEmTvJOJmCQLYQ9jff59L7HR6CuY1368 +dHz4pH4if2OmYf3GABcQqunXe/1VMRvdWQbXERbbHYuGstY11bHfc0t6zRhC3SqK +WCXp+1q79mUfV6hwOPP+skdoOFi0CDwvWXFkXvrri/Iu3G+YUIxhStkvJlZtdK2O +pwX7sGc722QqIFT0okcrvS4ksRgbtmbWURV2sQIDAQABAoIBAD9ITw4sBw0A95Qx +NTnBQsJBWDgp5TZaECc+51PjAW/8rCMtsYQ+JdBGrlnYuh3hYwR6iPhWHQpCax52 +DhbXUgP0hC96CqSSiF5OgmErUJmbfhyW/RMrCnJDKlFXXnqJJUR1/MTNyIVGg0Tn +hoQe3pLi26XS9y0QP7F919fWn2tnSbNaMrnMQoH4gD0GfrAH20zhFsAvmaNY7QGv +FVqUAIQLVDwnHGiLnEWgmOsA8G9NgwHaB3m20Eqmau6WbWbefGsaglhz2/t1yVZt +jDZBXSfb7FJpUd0MqXB56oe7ScUfk/S1pG5vv/DEXuN18juMhcYQE3gHJg08O/ys +6J23DkkCgYEA9uSQJiMZx0SFGE91gCDHy0vKddt15bOq2eD6t2RMZ01aRoic6oAj +Ox/co2zLNKaFjD+SZhqRCXcSUopV8zjzcZw7FXWc9w9qjuGTaar3v0dduyAd6SNl +l+iiLfc7m1P5ZJt8styTm7UKi6ZortuxxVDq+uRqw5vEAYHyaHfdUdMCgYEAwkA5 +zCLR8pqRDjGcMDd3wxjOuaXdhjRv/Q1V9bcxamvsOFrL9VvvfrVoFrmgW8uc1vSW +q08b5W0N8y6R+4OFkWh9RW+RQEC1ScJ1xLB4FDrHWUit5oCCTlU8MxOH6Z7dnKF7 +IaKqRSXYDiFcRjgX8OmiY36+RGCg/NhrZPXU/usCgYB2QNeCTMGmHf2ZvUVMT2ci +ynR4qtr3YGzu2sF7YYDu4a/hpMFz2FgHk6U8aXmTCEdi+3gFi6f4Xp3aBwRP8PE5 +c1khWCoMc8fYE9dlf0wWw/nzDNkGt7uLXROA7LASH9CODaNWkCvrInWgmd3+EqA9 +IBrMfOdeNZdWGYWf3/mTnQKBgBSxDLcO+ngyOMNfS3jjX7F7ggHKhkdWbJtzATYU +VoI0jswN9+3h5igEINtS2J8OmF1ZXndPDPSqYjMGQeydvOuZSsbLtJg4jFcYDL1v +4sqc3EFJeaS1CaxYbfou2WpsTsTxepFBp4uIFXSj3xVnwj0aVf7tME1OUbe9QetW +OGMTAoGAUSjmIjzlWW6sbCBoJJicklQmamvGd07Vg3TxP4NE1Xh7DZ4LNM3saarF +Z57gAqocWfb+RExEPnCbmhjws2THU2oUheLU2f5llubf74rrWZHzkGBnA+KCs9UT +lL1fs/fg9NjSdggkw7M1Unt9iAufZhsaGri+QTD3Jmm64q4LEKI= +-----END RSA PRIVATE KEY----- diff --git a/test/unitTests/testAssets/public.pem b/test/unitTests/testAssets/public.pem new file mode 100644 index 000000000..40f77a623 --- /dev/null +++ b/test/unitTests/testAssets/public.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDcTCCAlmgAwIBAgIJAIUrQVQLJdmOMA0GCSqGSIb3DQEBCwUAME8xCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJXQTEQMA4GA1UEBwwHU2VhdHRsZTEhMB8GA1UECgwY +SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMB4XDTE4MDQxMTIyMTM0OFoXDTIxMDEw +NTIyMTM0OFowTzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMRAwDgYDVQQHDAdT +ZWF0dGxlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC7VyYes/KBYFVavSeDXuYRfFb6OCt7 +m+Lf0YsNdtJzMzqsUnszRSb/hSFJa21yc/jxy2DSiybhxBGuAL8XTkwJDI9YWarT +9JX4K3zoozWNGlbSeXB+pfuTFl8DpopHdclv882Ff8usowKPeFOUSZO8k4mYJAth +D2N9/n0vsdHoK5jXfrx0fPikfiJ/Y6Zh/cYAFxCq6dd7/VUxG91ZBtcRFtsdi4ay +1jXVsd9zS3rNGELdKopYJen7Wrv2ZR9XqHA48/6yR2g4WLQIPC9ZcWRe+uuL8i7c +b5hQjGFK2S8mVm10rY6nBfuwZzvbZCogVPSiRyu9LiSxGBu2ZtZRFXaxAgMBAAGj +UDBOMB0GA1UdDgQWBBRtfbc2ltDm/xqKyiTH7o7gFFqTGDAfBgNVHSMEGDAWgBRt +fbc2ltDm/xqKyiTH7o7gFFqTGDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQCjj08qk3NBi9XcALdxL0vQSBODeflRlKgQss1hxwp2RJNVNjVXchk4DPxb +87EE9dRO9soARrHWWdl3teqnIGPSmJ65FLSWQzkeX8duQDFLwT6JRazl2sonBLeO +NUhodmrmw+Yb7ABXhITLSyuC7iPCwBVTUGbY1BGYrh9GJM5qjf4+krI+2MMw3N/o +yHmPUl4q4mmBOas+g2ngyv7B2GJL1FEnMPWJToAQdIct+K9orrEfp69sK8ydBxb/ +4FD2skruTux7AkXSfQb5WEJKswvs7yQFxVBsImiRO/EIwwCU98bsyVcPbVlBmUle +T7h9SDA3e33HulS38W625h1wWi4J +-----END CERTIFICATE----- From c74aecf639b40566734a3aa8f8c898bb0540c6e6 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Thu, 12 Apr 2018 11:10:41 -0700 Subject: [PATCH 10/64] Refactoring packages-1 --- src/CSharpExtDownloader.ts | 25 +- src/downloader.helper.ts | 19 -- src/omnisharp/OmnisharpDownloader.ts | 21 +- src/packageManager/PackageDownloader.ts | 145 ++++++++++ src/packageManager/PackageInstaller.ts | 109 ++++++++ src/packageManager/PackageManager.ts | 64 +++++ src/packageManager/packages.ts | 97 +++++++ src/{ => packageManager}/proxy.ts | 0 src/packages.ts | 345 ------------------------ 9 files changed, 429 insertions(+), 396 deletions(-) delete mode 100644 src/downloader.helper.ts create mode 100644 src/packageManager/PackageDownloader.ts create mode 100644 src/packageManager/PackageInstaller.ts create mode 100644 src/packageManager/PackageManager.ts create mode 100644 src/packageManager/packages.ts rename src/{ => packageManager}/proxy.ts (100%) delete mode 100644 src/packages.ts diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 6f6a2bd11..8d3878347 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -4,32 +4,25 @@ *--------------------------------------------------------------------------------------------*/ import * as util from './common'; -import { GetNetworkConfiguration, defaultPackageManagerFactory, IPackageManagerFactory } from './downloader.helper'; -import { PackageManager, Package } from './packages'; import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; import { vscode } from './vscodeAdapter'; +import { PackageManager } from './packageManager/PackageManager'; +import { Package } from './packageManager/packages'; /* * Class used to download the runtime dependencies of the C# Extension */ export class CSharpExtDownloader { - private proxy: string; - private strictSSL: boolean; private packageManager: PackageManager; public constructor( - vscode: vscode, + private vscode: vscode, private eventStream: EventStream, private packageJSON: any, - private platformInfo: PlatformInformation, - packageManagerFactory: IPackageManagerFactory = defaultPackageManagerFactory) { - - let networkConfiguration = GetNetworkConfiguration(vscode); - this.proxy = networkConfiguration.Proxy; - this.strictSSL = networkConfiguration.StrictSSL; - this.packageManager = packageManagerFactory(this.platformInfo); + private platformInfo: PlatformInformation) { + this.packageManager = new PackageManager(); } public async installRuntimeDependencies(): Promise { @@ -45,12 +38,10 @@ export class CSharpExtDownloader { let runTimeDependencies = this.GetRunTimeDependenciesPackages(); - installationStage = 'downloadPackages'; - let downloadedPackages = await this.packageManager.DownloadPackages(runTimeDependencies, this.eventStream, this.proxy, this.strictSSL); - - installationStage = 'installPackages'; - await this.packageManager.InstallPackages(downloadedPackages,this.eventStream); + installationStage = 'downloadAndInstallPackages'; + await this.packageManager.DownloadAndInstallPackages(runTimeDependencies, this.vscode, this.platformInfo, this.eventStream); + // We probably dont need the install.Lock thing now as we are directly testing the package test path installationStage = 'touchLockFile'; await util.touchInstallFile(util.InstallFileType.Lock); diff --git a/src/downloader.helper.ts b/src/downloader.helper.ts deleted file mode 100644 index f0e8288bc..000000000 --- a/src/downloader.helper.ts +++ /dev/null @@ -1,19 +0,0 @@ -/*--------------------------------------------------------------------------------------------- -* Copyright (c) Microsoft Corporation. All rights reserved. -* Licensed under the MIT License. See License.txt in the project root for license information. -*--------------------------------------------------------------------------------------------*/ -import { vscode } from "./vscodeAdapter"; -import { PlatformInformation } from "./platform"; -import { PackageManager } from "./packages"; - -export function GetNetworkConfiguration(vscode: vscode) { - const config = vscode.workspace.getConfiguration(); - const proxy = config.get('http.proxy'); - const strictSSL = config.get('http.proxyStrictSSL', true); - return { Proxy: proxy, StrictSSL: strictSSL }; -} - -export const defaultPackageManagerFactory: IPackageManagerFactory = (platformInfo) => new PackageManager(platformInfo); -export interface IPackageManagerFactory { - (platformInfo: PlatformInformation): PackageManager; -} diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 15c331668..ee6e96c9f 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -3,13 +3,12 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { GetNetworkConfiguration, IPackageManagerFactory, defaultPackageManagerFactory } from '../downloader.helper'; import { GetPackagesFromVersion, GetVersionFilePackage } from './OmnisharpPackageCreator'; -import { PackageManager } from '../packages'; import { PlatformInformation } from '../platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; import { EventStream } from '../EventStream'; import { vscode } from '../vscodeAdapter'; +import { PackageManager } from '../packageManager/PackageManager'; export class OmnisharpDownloader { @@ -18,16 +17,12 @@ export class OmnisharpDownloader { private packageManager: PackageManager; public constructor( - vscode: vscode, + private vscode: vscode, private eventStream: EventStream, private packageJSON: any, - private platformInfo: PlatformInformation, - packageManagerFactory: IPackageManagerFactory = defaultPackageManagerFactory) { + private platformInfo: PlatformInformation) { - let networkConfiguration = GetNetworkConfiguration(vscode); - this.proxy = networkConfiguration.Proxy; - this.strictSSL = networkConfiguration.StrictSSL; - this.packageManager = packageManagerFactory(this.platformInfo); + this.packageManager = new PackageManager(); } public async DownloadAndInstallOmnisharp(version: string, serverUrl: string, installPath: string) { @@ -40,12 +35,8 @@ export class OmnisharpDownloader { installationStage = 'getPackageInfo'; let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); - installationStage = 'downloadPackages'; - - let downloadedPackages = await this.packageManager.DownloadPackages(packages, this.eventStream, this.proxy, this.strictSSL); - - installationStage = 'installPackages'; - await this.packageManager.InstallPackages(downloadedPackages, this.eventStream); + installationStage = 'downloadAndInstallPackages'; + await this.packageManager.DownloadAndInstallPackages(packages, this.vscode, this.platformInfo, this.eventStream); this.eventStream.post(new InstallationSuccess()); } diff --git a/src/packageManager/PackageDownloader.ts b/src/packageManager/PackageDownloader.ts new file mode 100644 index 000000000..400b18af9 --- /dev/null +++ b/src/packageManager/PackageDownloader.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as tmp from 'tmp'; +import * as https from 'https'; +import * as util from '../common'; +import * as fs from 'fs'; +import { EventStream } from "../EventStream"; +import { DownloadSuccess, DownloadStart, DownloadFallBack, DownloadFailure, DownloadProgress, DownloadSizeObtained } from "../omnisharp/loggingEvents"; +import { Package, PackageError } from "./packages"; +import { parse as parseUrl } from 'url'; +import { getProxyAgent } from './proxy'; +import { vscode } from '../vscodeAdapter'; + +export class PackageDownloader { + constructor() { + // Ensure our temp files get cleaned up in case of error. + tmp.setGracefulCleanup(); + } + + public async DownloadPackage(description: string, url: string, fallbackUrl: string, vscode: vscode, eventStream: EventStream) { + const config = vscode.workspace.getConfiguration(); + const proxy = config.get('http.proxy'); + const strictSSL = config.get('http.proxyStrictSSL', true); + return downloadPackage(pkg, eventStream, proxy, strictSSL); + } +} + +// We dont need this as we are already doing this check in the package manager +/*async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { + let exists = await doesPackageTestPathExist(pkg); + if (!exists) { + return downloadPackage(pkg, eventStream, proxy, strictSSL); + } else { + eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); + return pkg; + } +}*/ + +async function downloadPackage(description: string, url: string, fallbackUrl: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { + eventStream.post(new DownloadStart(description)); + const tmpResult = await new Promise((resolve, reject) => { + tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { + if (err) { + return reject(new PackageError('Error from tmp.file', description, err)); + } + + resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); + }); + }); + + try { + await downloadFile(tmpResult, description,url, eventStream, proxy, strictSSL); + eventStream.post(new DownloadSuccess(` Done!`)); + } + catch (primaryUrlError) { + // If the package has a fallback Url, and downloading from the primary Url failed, try again from + // the fallback. This is used for debugger packages as some users have had issues downloading from + // the CDN link + if (fallbackUrl) { + eventStream.post(new DownloadFallBack(fallbackUrl)); + try { + await downloadFile(tmpFile, description, fallbackUrl, eventStream, proxy, strictSSL); + eventStream.post(new DownloadSuccess(' Done!')); + } + catch (fallbackUrlError) { + throw primaryUrlError; + } + } + else { + throw primaryUrlError; + } + } +} + +async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string, urlString: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { + const url = parseUrl(urlString); + + const options: https.RequestOptions = { + host: url.hostname, + path: url.path, + agent: getProxyAgent(url, proxy, strictSSL), + port: url.port, + rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true + }; + + return new Promise((resolve, reject) => { + if (!tmpFile || tmpFile.fd == 0) { + return reject(new PackageError("Temporary package file unavailable", pkg)); + } + + let request = https.request(options, response => { + if (response.statusCode === 301 || response.statusCode === 302) { + // Redirect - download from new location + return resolve(downloadFile(tmpFile, description, response.headers.location, eventStream, proxy, strictSSL)); + } + + if (response.statusCode != 200) { + // Download failed - print error message + eventStream.post(new DownloadFailure(`failed (error code '${response.statusCode}')`)); + return reject(new PackageError(response.statusCode.toString(), description)); + } + + // Downloading - hook up events + let packageSize = parseInt(response.headers['content-length'], 10); + let downloadedBytes = 0; + let downloadPercentage = 0; + let tmpFile = fs.createWriteStream(null, { fd: tmpFile.fd }); + + eventStream.post(new DownloadSizeObtained(packageSize)); + + response.on('data', data => { + downloadedBytes += data.length; + + // Update status bar item with percentage + let newPercentage = Math.ceil(100 * (downloadedBytes / packageSize)); + if (newPercentage !== downloadPercentage) { + downloadPercentage = newPercentage; + eventStream.post(new DownloadProgress(downloadPercentage, description)); + } + }); + + response.on('end', () => { + resolve(); + }); + + response.on('error', err => { + reject(new PackageError(`Reponse error: ${err.message || 'NONE'}`, description, err)); + }); + + // Begin piping data from the response to the package file + response.pipe(tmpFile, { end: false }); + }); + + request.on('error', err => { + console.log(err); + reject(new PackageError(`Request error: ${err.message || 'NONE'}`, description, err)); + }); + + // Execute the request + request.end(); + }); +} \ No newline at end of file diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts new file mode 100644 index 000000000..dc8c92492 --- /dev/null +++ b/src/packageManager/PackageInstaller.ts @@ -0,0 +1,109 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Package, PackageError, getPackageTestPath, getBaseInstallPath } from "./packages"; +import { EventStream } from "../EventStream"; +import { InstallationProgress } from "../omnisharp/loggingEvents"; +import * as fs from 'fs'; +import * as mkdirp from 'mkdirp'; +import * as yauzl from 'yauzl'; +import * as path from 'path'; + +export class PackageInstaller { + constructor() { } + + public InstallPackage(pkg: Package, eventStream: EventStream): Promise { + const installationStage = 'installPackages'; + if (!pkg.tmpFile) { + // Download of this package was skipped, so there is nothing to install + return Promise.resolve(); + } + + resolvePackageBinaries(pkg); + eventStream.post(new InstallationProgress(installationStage, pkg.description)); + + return new Promise((resolve, baseReject) => { + const reject = (err: any) => { + // If anything goes wrong with unzip, make sure we delete the test path (if there is one) + // so we will retry again later + const testPath = getPackageTestPath(pkg); + if (testPath) { + fs.unlink(testPath, unlinkErr => { + baseReject(err); + }); + } else { + baseReject(err); + } + }; + + if (pkg.tmpFile.fd == 0) { + return reject(new PackageError('Downloaded file unavailable', pkg)); + } + + yauzl.fromFd(pkg.tmpFile.fd, { lazyEntries: true }, (err, zipFile) => { + if (err) { + return reject(new PackageError('Immediate zip file error', pkg, err)); + } + + zipFile.readEntry(); + + zipFile.on('entry', (entry: yauzl.Entry) => { + let absoluteEntryPath = path.resolve(getBaseInstallPath(pkg), entry.fileName); + + if (entry.fileName.endsWith('/')) { + // Directory - create it + mkdirp(absoluteEntryPath, { mode: 0o775 }, err => { + if (err) { + return reject(new PackageError('Error creating directory for zip directory entry:' + err.code || '', pkg, err)); + } + + zipFile.readEntry(); + }); + } + else { + // File - extract it + zipFile.openReadStream(entry, (err, readStream) => { + if (err) { + return reject(new PackageError('Error reading zip stream', pkg, err)); + } + + mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => { + if (err) { + return reject(new PackageError('Error creating directory for zip file entry', pkg, err)); + } + + // Make sure executable files have correct permissions when extracted + let fileMode = pkg.binaries && pkg.binaries.indexOf(absoluteEntryPath) !== -1 + ? 0o755 + : 0o664; + + readStream.pipe(fs.createWriteStream(absoluteEntryPath, { mode: fileMode })); + readStream.on('end', () => zipFile.readEntry()); + }); + }); + } + }); + + zipFile.on('end', () => { + resolve(); + }); + + zipFile.on('error', err => { + reject(new PackageError('Zip File Error:' + err.code || '', pkg, err)); + }); + }); + }).then(() => { + // Clean up temp file + pkg.tmpFile.removeCallback(); + }); + } +} + +function resolvePackageBinaries(pkg: Package) { + // Convert relative binary paths to absolute + if (pkg.binaries) { + pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); + } +} \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts new file mode 100644 index 000000000..88ee3841f --- /dev/null +++ b/src/packageManager/PackageManager.ts @@ -0,0 +1,64 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PlatformInformation } from "../platform"; +import { Package, PackageError, doesPackageTestPathExist } from './packages'; +import { PackageDownloader } from './PackageDownloader'; +import { PackageInstaller } from './PackageInstaller'; +import { vscode } from '../vscodeAdapter'; +import { EventStream } from '../EventStream'; + +export class PackageManager { + public constructor() { + } + + private async filterPackages(packages: Package[], platformInfo: PlatformInformation) { + let platformPackages = await filterPlatformPackages(packages, platformInfo); + return filterAlreadyInstalledPackages(platformPackages); + } + + public async DownloadAndInstallPackages(packages: Package[], vscode: vscode, platformInfo: PlatformInformation, eventStream: EventStream) { + let filteredPackages = await this.filterPackages(packages, platformInfo); + if (filteredPackages) { + let downloader = new PackageDownloader(); + let installer = new PackageInstaller(); + for (let pkg of filteredPackages) { + await downloader.DownloadPackage(pkg, vscode, eventStream); + // see into error handling as well + await installer.InstallPackage(pkg, eventStream); + } + } + } +} + +async function filterPlatformPackages(packages: Package[], platformInfo: PlatformInformation) { + if (packages) { + return packages.filter(pkg => { + if (pkg.architectures && pkg.architectures.indexOf(platformInfo.architecture) === -1) { + return false; + } + + if (pkg.platforms && pkg.platforms.indexOf(platformInfo.platform) === -1) { + return false; + } + + return true; + }); + } + else { + throw new PackageError("Package manifest does not exist."); + } +} + +async function filterAlreadyInstalledPackages(packages: Package[]) { + return packages.filter(async pkg => { + if (await doesPackageTestPathExist(pkg)) { + return false; // do not download the already installed packages + } + else { + return true; + } + }); +} \ No newline at end of file diff --git a/src/packageManager/packages.ts b/src/packageManager/packages.ts new file mode 100644 index 000000000..4ed9cd33d --- /dev/null +++ b/src/packageManager/packages.ts @@ -0,0 +1,97 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as path from 'path'; +import * as tmp from 'tmp'; +import * as util from '../common'; + +export interface Package { + description: string; + url: string; + fallbackUrl?: string; + installPath?: string; + platforms: string[]; + architectures: string[]; + binaries: string[]; + tmpFile: tmp.SynchrounousResult; + platformId?: string; + + // Path to use to test if the package has already been installed + installTestPath?: string; +} + +export class PackageError extends Error { + // Do not put PII (personally identifiable information) in the 'message' field as it will be logged to telemetry + constructor(public message: string, + public packageDescription: string = null, + public innerError: any = null) { + super(message); + } +} + +/*export class PackageManager { + public constructor() { } + + public async DownloadPackages(packages: Package[], eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { + let packagesToDownload = await this.GetPackages(packages); + for(let pkg of packagesToDownload){ + await maybeDownloadPackage(pkg, eventStream, proxy, strictSSL); + } + return packagesToDownload; + } + + public async InstallPackages(packages: Package[], eventStream: EventStream): Promise { + let packagesToInstall = await this.GetPackages(packages); + return util.buildPromiseChain(packagesToInstall, async pkg => installPackage(pkg, eventStream)); + } + + + + public async GetLatestVersionFromFile(eventStream: EventStream, proxy: string, strictSSL: boolean, filePackage: Package): Promise { + try { + let latestVersion: string; + await maybeDownloadPackage(filePackage, eventStream, proxy, strictSSL); + if (filePackage.tmpFile) { + latestVersion = fs.readFileSync(filePackage.tmpFile.name, 'utf8'); + //Delete the temporary file created + filePackage.tmpFile.removeCallback(); + } + + return latestVersion; + } + catch (error) { + throw new Error(`Could not download the latest version file due to ${error.toString()}`); + } + } +}*/ + +export function getBaseInstallPath(pkg: Package): string { + let basePath = util.getExtensionPath(); + if (pkg.installPath) { + basePath = path.join(basePath, pkg.installPath); + } + + return basePath; +} + +export async function doesPackageTestPathExist(pkg: Package): Promise { + const testPath = getPackageTestPath(pkg); + if (testPath) { + return util.fileExists(testPath); + } + else { + return Promise.resolve(false); + } +} + +export function getPackageTestPath(pkg: Package): string { + if (pkg.installTestPath) { + return path.join(util.getExtensionPath(), pkg.installTestPath); + } + else { + return null; + } +} + diff --git a/src/proxy.ts b/src/packageManager/proxy.ts similarity index 100% rename from src/proxy.ts rename to src/packageManager/proxy.ts diff --git a/src/packages.ts b/src/packages.ts deleted file mode 100644 index 520f8759a..000000000 --- a/src/packages.ts +++ /dev/null @@ -1,345 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as fs from 'fs'; -import * as https from 'https'; -import * as mkdirp from 'mkdirp'; -import * as path from 'path'; -import * as tmp from 'tmp'; -import { parse as parseUrl } from 'url'; -import * as yauzl from 'yauzl'; -import * as util from './common'; -import { PlatformInformation } from './platform'; -import { getProxyAgent } from './proxy'; -import { DownloadSuccess, DownloadStart, DownloadFailure, DownloadProgress, InstallationProgress, DownloadFallBack, DownloadSizeObtained } from './omnisharp/loggingEvents'; -import { EventStream } from './EventStream'; - -export interface Package { - description: string; - url: string; - fallbackUrl?: string; - installPath?: string; - platforms: string[]; - architectures: string[]; - binaries: string[]; - tmpFile: tmp.SynchrounousResult; - platformId?: string; - - // Path to use to test if the package has already been installed - installTestPath?: string; -} - -export class PackageError extends Error { - // Do not put PII (personally identifiable information) in the 'message' field as it will be logged to telemetry - constructor(public message: string, - public pkg: Package = null, - public innerError: any = null) { - super(message); - } -} - -export class PackageManager { - public constructor( - private platformInfo: PlatformInformation) { - // Ensure our temp files get cleaned up in case of error. - tmp.setGracefulCleanup(); - } - - public async DownloadPackages(packages: Package[], eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let packagesToDownload = await this.GetPackages(packages); - for(let pkg of packagesToDownload){ - await maybeDownloadPackage(pkg, eventStream, proxy, strictSSL); - } - return packagesToDownload; - } - - public async InstallPackages(packages: Package[], eventStream: EventStream): Promise { - let packagesToInstall = await this.GetPackages(packages); - return util.buildPromiseChain(packagesToInstall, async pkg => installPackage(pkg, eventStream)); - } - - public async GetPackages(packages: Package[]) { - let filteredPackages = await this.FilterPackages(packages); - resolvePackageBinaries(filteredPackages); - return filteredPackages; - } - - private async FilterPackages(packages: Package[]) { - if (packages) { - return packages.filter(pkg => { - if (pkg.architectures && pkg.architectures.indexOf(this.platformInfo.architecture) === -1) { - return false; - } - - if (pkg.platforms && pkg.platforms.indexOf(this.platformInfo.platform) === -1) { - return false; - } - - return true; - }); - } - else { - throw new PackageError("Package manifest does not exist."); - } - } - - public async GetLatestVersionFromFile(eventStream: EventStream, proxy: string, strictSSL: boolean, filePackage: Package): Promise { - try { - let latestVersion: string; - await maybeDownloadPackage(filePackage, eventStream, proxy, strictSSL); - if (filePackage.tmpFile) { - latestVersion = fs.readFileSync(filePackage.tmpFile.name, 'utf8'); - //Delete the temporary file created - filePackage.tmpFile.removeCallback(); - } - - return latestVersion; - } - catch (error) { - throw new Error(`Could not download the latest version file due to ${error.toString()}`); - } - } -} - -function resolvePackageBinaries(packages: Package[]) { - // Convert relative binary paths to absolute - if (packages) { - for (let pkg of packages) { - if (pkg.binaries) { - pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); - } - } - } -} - -function getBaseInstallPath(pkg: Package): string { - let basePath = util.getExtensionPath(); - if (pkg.installPath) { - basePath = path.join(basePath, pkg.installPath); - } - - return basePath; -} - -async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let exists = await doesPackageTestPathExist(pkg); - if (!exists) { - return downloadPackage(pkg, eventStream, proxy, strictSSL); - } else { - eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); - return pkg; - } -} - -async function downloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - eventStream.post(new DownloadStart(pkg.description)); - const tmpResult = await new Promise((resolve, reject) => { - tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { - if (err) { - return reject(new PackageError('Error from tmp.file', pkg, err)); - } - - resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); - }); - }); - - pkg.tmpFile = tmpResult; - - try { - await downloadFile(pkg.url, pkg, eventStream, proxy, strictSSL); - eventStream.post(new DownloadSuccess(` Done!`)); - return pkg; //return the downloaded package - } - catch (primaryUrlError) { - // If the package has a fallback Url, and downloading from the primary Url failed, try again from - // the fallback. This is used for debugger packages as some users have had issues downloading from - // the CDN link - if (pkg.fallbackUrl) { - eventStream.post(new DownloadFallBack(pkg.fallbackUrl)); - try { - await downloadFile(pkg.fallbackUrl, pkg, eventStream, proxy, strictSSL); - eventStream.post(new DownloadSuccess(' Done!')); - return pkg; - } - catch (fallbackUrlError) { - throw primaryUrlError; - } - } - else { - throw primaryUrlError; - } - } -} - -async function downloadFile(urlString: string, pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - const url = parseUrl(urlString); - - const options: https.RequestOptions = { - host: url.hostname, - path: url.path, - agent: getProxyAgent(url, proxy, strictSSL), - port: url.port, - rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true - }; - - return new Promise((resolve, reject) => { - if (!pkg.tmpFile || pkg.tmpFile.fd == 0) { - return reject(new PackageError("Temporary package file unavailable", pkg)); - } - - let request = https.request(options, response => { - if (response.statusCode === 301 || response.statusCode === 302) { - // Redirect - download from new location - return resolve(downloadFile(response.headers.location, pkg, eventStream, proxy, strictSSL)); - } - - if (response.statusCode != 200) { - // Download failed - print error message - eventStream.post(new DownloadFailure(`failed (error code '${response.statusCode}')`)); - return reject(new PackageError(response.statusCode.toString(), pkg)); - } - - // Downloading - hook up events - let packageSize = parseInt(response.headers['content-length'], 10); - let downloadedBytes = 0; - let downloadPercentage = 0; - let tmpFile = fs.createWriteStream(null, { fd: pkg.tmpFile.fd }); - - eventStream.post(new DownloadSizeObtained(packageSize)); - - response.on('data', data => { - downloadedBytes += data.length; - - // Update status bar item with percentage - let newPercentage = Math.ceil(100 * (downloadedBytes / packageSize)); - if (newPercentage !== downloadPercentage) { - downloadPercentage = newPercentage; - eventStream.post(new DownloadProgress(downloadPercentage, pkg.description)); - } - }); - - response.on('end', () => { - resolve(); - }); - - response.on('error', err => { - reject(new PackageError(`Reponse error: ${err.message || 'NONE'}`, pkg, err)); - }); - - // Begin piping data from the response to the package file - response.pipe(tmpFile, { end: false }); - }); - - request.on('error', err => { - console.log(err); - reject(new PackageError(`Request error: ${err.message || 'NONE'}`, pkg, err)); - }); - - // Execute the request - request.end(); - }); -} - -async function installPackage(pkg: Package, eventStream: EventStream): Promise { - const installationStage = 'installPackages'; - if (!pkg.tmpFile) { - // Download of this package was skipped, so there is nothing to install - return Promise.resolve(); - } - - eventStream.post(new InstallationProgress(installationStage, pkg.description)); - - return new Promise((resolve, baseReject) => { - const reject = (err: any) => { - // If anything goes wrong with unzip, make sure we delete the test path (if there is one) - // so we will retry again later - const testPath = getPackageTestPath(pkg); - if (testPath) { - fs.unlink(testPath, unlinkErr => { - baseReject(err); - }); - } else { - baseReject(err); - } - }; - - if (pkg.tmpFile.fd == 0) { - return reject(new PackageError('Downloaded file unavailable', pkg)); - } - - yauzl.fromFd(pkg.tmpFile.fd, { lazyEntries: true }, (err, zipFile) => { - if (err) { - return reject(new PackageError('Immediate zip file error', pkg, err)); - } - - zipFile.readEntry(); - - zipFile.on('entry', (entry: yauzl.Entry) => { - let absoluteEntryPath = path.resolve(getBaseInstallPath(pkg), entry.fileName); - - if (entry.fileName.endsWith('/')) { - // Directory - create it - mkdirp(absoluteEntryPath, { mode: 0o775 }, err => { - if (err) { - return reject(new PackageError('Error creating directory for zip directory entry:' + err.code || '', pkg, err)); - } - - zipFile.readEntry(); - }); - } - else { - // File - extract it - zipFile.openReadStream(entry, (err, readStream) => { - if (err) { - return reject(new PackageError('Error reading zip stream', pkg, err)); - } - - mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => { - if (err) { - return reject(new PackageError('Error creating directory for zip file entry', pkg, err)); - } - - // Make sure executable files have correct permissions when extracted - let fileMode = pkg.binaries && pkg.binaries.indexOf(absoluteEntryPath) !== -1 - ? 0o755 - : 0o664; - - readStream.pipe(fs.createWriteStream(absoluteEntryPath, { mode: fileMode })); - readStream.on('end', () => zipFile.readEntry()); - }); - }); - } - }); - - zipFile.on('end', () => { - resolve(); - }); - - zipFile.on('error', err => { - reject(new PackageError('Zip File Error:' + err.code || '', pkg, err)); - }); - }); - }).then(() => { - // Clean up temp file - pkg.tmpFile.removeCallback(); - }); -} - -async function doesPackageTestPathExist(pkg: Package): Promise { - const testPath = getPackageTestPath(pkg); - if (testPath) { - return util.fileExists(testPath); - } else { - return Promise.resolve(false); - } -} - -function getPackageTestPath(pkg: Package): string { - if (pkg.installTestPath) { - return path.join(util.getExtensionPath(), pkg.installTestPath); - } else { - return null; - } -} \ No newline at end of file From 3e58f13ac48a5b0f1a5a7bc9159d814b7b906857 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 12 Apr 2018 19:00:03 -0700 Subject: [PATCH 11/64] Test for the downloader running using the mock server --- .../Packages/PackageDownloader.test.ts | 74 +++++++++++++++++ test/unitTests/packageManager.test.ts | 81 ------------------- 2 files changed, 74 insertions(+), 81 deletions(-) create mode 100644 test/unitTests/Packages/PackageDownloader.test.ts delete mode 100644 test/unitTests/packageManager.test.ts diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts new file mode 100644 index 000000000..5ac092fd0 --- /dev/null +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import * as util from '../../../src/common'; +import { rimraf } from 'async-file'; +import { EventStream } from '../../../src/EventStream'; +import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; +import NetworkSettings from '../../../src/NetworkSettings'; + +let ServerMock = require("mock-http-server"); +const chai = require("chai"); +chai.use(require("chai-as-promised")); +let expect = chai.expect; + +suite("PackageDownloader : The package is downloaded ", () => { + let server = new ServerMock({ host: "127.0.0.1", port: 9001 }, + { + host: "localhost", + port: 9002, + key: fs.readFileSync("test/unitTests/testAssets/private.pem"), + cert: fs.readFileSync("test/unitTests/testAssets/public.pem") + }); + + let tmpFile: tmp.SynchrounousResult = null; + const eventStream = new EventStream(); + const serverUrl = "https://127.0.0.1:9002"; + + setup(function (done) { + server.start(done); + tmpFile = tmp.fileSync(); + util.setExtensionPath(tmpFile.name); + }); + + teardown(function (done) { + server.stop(done); + }); + + test('Packages are downloaded from the specified server url and installed at the specified path', async () => { + server.on({ + method: 'GET', + path: '/resource', + reply: { + status: 200, + headers: { "content-type": "text/plain" }, + body: "something" + } + }); + + let description = "Latest version information file"; + let url = `${serverUrl}/resource`; + + try { + await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); + } + catch (error) { + console.log(error); + } + + const stats = fs.statSync(tmpFile.name); + expect(stats.size).to.not.equal(0); + }); + + teardown(async () => { + if (tmpFile) { + await rimraf(tmpFile.name); + } + + tmpFile = null; + }); +}); diff --git a/test/unitTests/packageManager.test.ts b/test/unitTests/packageManager.test.ts deleted file mode 100644 index 6d07e16f0..000000000 --- a/test/unitTests/packageManager.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as tmp from 'tmp'; -import * as util from '../../src/common'; -import { rimraf } from 'async-file'; -import { PlatformInformation } from '../../src/platform'; -import { EventStream } from '../../src/EventStream'; -import { PackageManager, Package } from '../../src/packages'; -import * as fs from 'fs'; - -let ServerMock = require("mock-http-server"); -const chai = require("chai"); -chai.use(require("chai-as-promised")); -let expect = chai.expect; - -suite("DownloadAndInstallExperimentalVersion : Gets the version packages, downloads and installs them", () => { - let server = new ServerMock({ host: "127.0.0.1", port: 9001 }, - { - host: "localhost", - port: 9002, - key: fs.readFileSync("test/unitTests/testAssets/private.pem"), - cert: fs.readFileSync("test/unitTests/testAssets/public.pem") - }); - let tmpDir: tmp.SynchrounousResult = null; - const platformInfo = new PlatformInformation("win32", "x86"); - const eventStream = new EventStream(); - const manager = new PackageManager(platformInfo); - //const serverUrl = "https://roslynomnisharp.blob.core.windows.net"; - const serverUrl = "https://127.0.0.1:9002"; - - setup(function (done) { - server.start(done); - tmpDir = tmp.dirSync(); - util.setExtensionPath(tmpDir.name); - }); - - teardown(function (done) { - server.stop(done); - }); - - test('Packages are downloaded from the specified server url and installed at the specified path', async() =>{ - console.log("hello1"); - server.on({ - method: 'GET', - path: '/resource', - reply: { - status: 200, - headers: { "content-type": "text/plain" }, - body: "something" - } - }); - - let packages = Array({ - "description": "Latest version information file", - "url": `${serverUrl}/resource` - }); - console.log(packages[0].url); - let downloadedPackage : Package[]; - - try { - downloadedPackage = await manager.DownloadPackages(packages, eventStream, undefined, false); - } - catch (error) { - console.log(error); - } - - expect(downloadedPackage).to.not.be.empty; - expect(downloadedPackage[0].tmpFile).to.not.be.undefined; - }); - - teardown(async () => { - if (tmpDir) { - await rimraf(tmpDir.name); - } - - tmpDir = null; - }); -}); From 5565a2eea4034db6e171e694fa3ef793c8ed1674 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:43:14 -0700 Subject: [PATCH 12/64] Using network settings --- src/NetworkSettings.ts | 25 +++++++++++++++++++++++++ src/main.ts | 12 +++++++----- src/omnisharp/extension.ts | 6 +++--- src/omnisharp/server.ts | 5 +++-- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 src/NetworkSettings.ts diff --git a/src/NetworkSettings.ts b/src/NetworkSettings.ts new file mode 100644 index 000000000..2bbea1674 --- /dev/null +++ b/src/NetworkSettings.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { vscode } from "./vscodeAdapter"; + +export default class NetworkSettings { + constructor(public readonly proxy: string, public readonly strictSSL: boolean) { + } +} + +export interface NetworkSettingsProvider { + //to do: make this async + (): NetworkSettings; +} + +export function vscodeNetworkSettingsProvider(vscode: vscode): NetworkSettingsProvider { + return () => { + const config = vscode.workspace.getConfiguration(); + const proxy = config.get('http.proxy'); + const strictSSL = config.get('http.proxyStrictSSL', true); + return new NetworkSettings(proxy, strictSSL); + }; +} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 7c04f4357..3cb9a32c5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -28,6 +28,7 @@ import TelemetryReporter from 'vscode-extension-telemetry'; import { addJSONProviders } from './features/json/jsonContributions'; import { ProjectStatusBarObserver } from './observers/ProjectStatusBarObserver'; import CSharpExtensionExports from './CSharpExtensionExports'; +import { vscodeNetworkSettingsProvider, NetworkSettingsProvider } from './NetworkSettings'; export async function activate(context: vscode.ExtensionContext): Promise { @@ -90,10 +91,11 @@ export async function activate(context: vscode.ExtensionContext): Promise reporter); eventStream.subscribe(telemetryObserver.post); - let runtimeDependenciesExist = await ensureRuntimeDependencies(extension, eventStream, platformInfo); - + let networkSettingsProvider = vscodeNetworkSettingsProvider(vscode); + let runtimeDependenciesExist = await ensureRuntimeDependencies(extension, eventStream, platformInfo, networkSettingsProvider); + // activate language services - let omniSharpPromise = OmniSharp.activate(context, eventStream, extension.packageJSON, platformInfo); + let omniSharpPromise = OmniSharp.activate(context, eventStream, extension.packageJSON, platformInfo, networkSettingsProvider); // register JSON completion & hover providers for project.json context.subscriptions.push(addJSONProviders()); @@ -116,11 +118,11 @@ export async function activate(context: vscode.ExtensionContext): Promise, eventStream: EventStream, platformInfo: PlatformInformation): Promise { +async function ensureRuntimeDependencies(extension: vscode.Extension, eventStream: EventStream, platformInfo: PlatformInformation, networkSettingsProvider : NetworkSettingsProvider): Promise { return util.installFileExists(util.InstallFileType.Lock) .then(exists => { if (!exists) { - const downloader = new CSharpExtDownloader(vscode, eventStream, extension.packageJSON, platformInfo); + const downloader = new CSharpExtDownloader(networkSettingsProvider, eventStream, extension.packageJSON, platformInfo); return downloader.installRuntimeDependencies(); } else { return true; diff --git a/src/omnisharp/extension.ts b/src/omnisharp/extension.ts index e0c7e6bfb..dad99cb14 100644 --- a/src/omnisharp/extension.ts +++ b/src/omnisharp/extension.ts @@ -31,18 +31,18 @@ import registerCommands from '../features/commands'; import { PlatformInformation } from '../platform'; import { ProjectJsonDeprecatedWarning, OmnisharpStart } from './loggingEvents'; import { EventStream } from '../EventStream'; +import { NetworkSettingsProvider } from '../NetworkSettings'; export let omnisharp: OmniSharpServer; -export async function activate(context: vscode.ExtensionContext, eventStream: EventStream, packageJSON: any, platformInfo: PlatformInformation) { +export async function activate(context: vscode.ExtensionContext, eventStream: EventStream, packageJSON: any, platformInfo: PlatformInformation, provider: NetworkSettingsProvider) { const documentSelector: vscode.DocumentSelector = { language: 'csharp', scheme: 'file' // only files from disk }; const options = Options.Read(); - const server = new OmniSharpServer(vscode, eventStream, packageJSON, platformInfo); - + const server = new OmniSharpServer(vscode, provider, eventStream, packageJSON, platformInfo); omnisharp = server; const advisor = new Advisor(server); // create before server is started const disposables: vscode.Disposable[] = []; diff --git a/src/omnisharp/server.ts b/src/omnisharp/server.ts index cc2d03513..346766824 100644 --- a/src/omnisharp/server.ts +++ b/src/omnisharp/server.ts @@ -24,6 +24,7 @@ import { OmnisharpDownloader } from './OmnisharpDownloader'; import * as ObservableEvents from './loggingEvents'; import { EventStream } from '../EventStream'; import { Disposable, CompositeDisposable, Subject } from 'rx'; +import { NetworkSettingsProvider } from '../NetworkSettings'; enum ServerState { Starting, @@ -89,11 +90,11 @@ export class OmniSharpServer { private firstUpdateProject: boolean; private vscode: vscode; - constructor(vscode: vscode, eventStream: EventStream, packageJSON: any, platformInfo: PlatformInformation) { + constructor(vscode: vscode, networkSettingsProvider: NetworkSettingsProvider, eventStream: EventStream, packageJSON: any, platformInfo: PlatformInformation) { this.eventStream = eventStream; this.vscode = vscode; this._requestQueue = new RequestQueueCollection(this.eventStream, 8, request => this._makeRequest(request)); - let downloader = new OmnisharpDownloader(this.vscode, this.eventStream, packageJSON, platformInfo); + let downloader = new OmnisharpDownloader(networkSettingsProvider, this.eventStream, packageJSON, platformInfo); this._omnisharpManager = new OmnisharpManager(downloader, platformInfo); this.updateProjectDebouncer.debounce(1500).subscribe((event) => { this.updateProjectInfo(); }); this.firstUpdateProject = true; From c8135b5accf4173dbda83abf0b69a72724e80b6c Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:44:45 -0700 Subject: [PATCH 13/64] Changing the package path --- src/observers/CsharpLoggerObserver.ts | 2 +- src/observers/TelemetryObserver.ts | 2 +- src/tools/UpdatePackageDependencies.ts | 2 +- test/unitTests/logging/CsharpLoggerObserver.test.ts | 2 +- test/unitTests/logging/TelemetryObserver.test.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/observers/CsharpLoggerObserver.ts b/src/observers/CsharpLoggerObserver.ts index 8940aae17..3105106ae 100644 --- a/src/observers/CsharpLoggerObserver.ts +++ b/src/observers/CsharpLoggerObserver.ts @@ -3,9 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { PackageError } from "../packages"; import { BaseLoggerObserver } from "./BaseLoggerObserver"; import * as Event from "../omnisharp/loggingEvents"; +import { PackageError } from "../packageManager/packages"; export class CsharpLoggerObserver extends BaseLoggerObserver { private dots: number; diff --git a/src/observers/TelemetryObserver.ts b/src/observers/TelemetryObserver.ts index 36173c76d..5d1b6d30b 100644 --- a/src/observers/TelemetryObserver.ts +++ b/src/observers/TelemetryObserver.ts @@ -3,9 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { PackageError } from "../packages"; import { PlatformInformation } from "../platform"; import { BaseEvent, PackageInstallation, InstallationFailure, InstallationSuccess, OmnisharpDelayTrackerEventMeasures, OmnisharpStart, TestExecutionCountReport, TelemetryEventWithMeasures } from "../omnisharp/loggingEvents"; +import { PackageError } from "../packageManager/packages"; export interface ITelemetryReporter { sendTelemetryEvent(eventName: string, properties?: { [key: string]: string }, measures?: { [key: string]: number }): void; diff --git a/src/tools/UpdatePackageDependencies.ts b/src/tools/UpdatePackageDependencies.ts index 678d2d1b2..b48378f43 100644 --- a/src/tools/UpdatePackageDependencies.ts +++ b/src/tools/UpdatePackageDependencies.ts @@ -5,8 +5,8 @@ import * as fs from 'fs'; import * as os from 'os'; +import { Package } from '../packageManager/packages'; -import { Package } from '../packages'; interface PackageJSONFile { diff --git a/test/unitTests/logging/CsharpLoggerObserver.test.ts b/test/unitTests/logging/CsharpLoggerObserver.test.ts index 7cb0ee908..19b13e8c6 100644 --- a/test/unitTests/logging/CsharpLoggerObserver.test.ts +++ b/test/unitTests/logging/CsharpLoggerObserver.test.ts @@ -7,8 +7,8 @@ import { should, expect } from 'chai'; import { getNullChannel } from '../testAssets/Fakes'; import { CsharpLoggerObserver } from '../../../src/observers/CsharpLoggerObserver'; import { PlatformInformation } from '../../../src/platform'; -import { PackageError } from '../../../src/packages'; import * as Event from '../../../src/omnisharp/loggingEvents'; +import { PackageError } from '../../../src/packageManager/packages'; suite("CsharpLoggerObserver", () => { suiteSetup(() => should()); diff --git a/test/unitTests/logging/TelemetryObserver.test.ts b/test/unitTests/logging/TelemetryObserver.test.ts index 50cb40478..022d82726 100644 --- a/test/unitTests/logging/TelemetryObserver.test.ts +++ b/test/unitTests/logging/TelemetryObserver.test.ts @@ -6,8 +6,8 @@ import { should, expect } from 'chai'; import { TelemetryObserver } from '../../../src/observers/TelemetryObserver'; import { PlatformInformation } from '../../../src/platform'; import { PackageInstallation, InstallationFailure, InstallationSuccess, TestExecutionCountReport, TelemetryEventWithMeasures, OmnisharpDelayTrackerEventMeasures, OmnisharpStart } from '../../../src/omnisharp/loggingEvents'; -import { PackageError, Package } from '../../../src/packages'; import { getNullTelemetryReporter } from '../testAssets/Fakes'; +import { PackageError, Package } from '../../../src/packageManager/packages'; const chai = require('chai'); chai.use(require('chai-arrays')); From 9afc8740dec45f2c276af258710011936ab866d5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:45:18 -0700 Subject: [PATCH 14/64] Dividing packages.ts into separate components --- src/packageManager/PackageDownloader.ts | 67 +++-------- src/packageManager/PackageFilterer.ts | 37 ++++++ src/packageManager/PackageInstaller.ts | 147 +++++++++++------------- src/packageManager/PackageManager.ts | 83 ++++++------- src/packageManager/TmpFileManager.ts | 37 ++++++ src/packageManager/packages.ts | 41 +++---- src/packageManager/proxy.ts | 2 +- 7 files changed, 205 insertions(+), 209 deletions(-) create mode 100644 src/packageManager/PackageFilterer.ts create mode 100644 src/packageManager/TmpFileManager.ts diff --git a/src/packageManager/PackageDownloader.ts b/src/packageManager/PackageDownloader.ts index 400b18af9..540d8dfae 100644 --- a/src/packageManager/PackageDownloader.ts +++ b/src/packageManager/PackageDownloader.ts @@ -3,56 +3,21 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as tmp from 'tmp'; import * as https from 'https'; import * as util from '../common'; import * as fs from 'fs'; import { EventStream } from "../EventStream"; import { DownloadSuccess, DownloadStart, DownloadFallBack, DownloadFailure, DownloadProgress, DownloadSizeObtained } from "../omnisharp/loggingEvents"; -import { Package, PackageError } from "./packages"; +import { NestedError } from "./packages"; import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; -import { vscode } from '../vscodeAdapter'; +import { NetworkSettingsProvider } from '../NetworkSettings'; -export class PackageDownloader { - constructor() { - // Ensure our temp files get cleaned up in case of error. - tmp.setGracefulCleanup(); - } - - public async DownloadPackage(description: string, url: string, fallbackUrl: string, vscode: vscode, eventStream: EventStream) { - const config = vscode.workspace.getConfiguration(); - const proxy = config.get('http.proxy'); - const strictSSL = config.get('http.proxyStrictSSL', true); - return downloadPackage(pkg, eventStream, proxy, strictSSL); - } -} - -// We dont need this as we are already doing this check in the package manager -/*async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let exists = await doesPackageTestPathExist(pkg); - if (!exists) { - return downloadPackage(pkg, eventStream, proxy, strictSSL); - } else { - eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`)); - return pkg; - } -}*/ - -async function downloadPackage(description: string, url: string, fallbackUrl: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { +export async function DownloadPackage(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, provider: NetworkSettingsProvider){ eventStream.post(new DownloadStart(description)); - const tmpResult = await new Promise((resolve, reject) => { - tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { - if (err) { - return reject(new PackageError('Error from tmp.file', description, err)); - } - - resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); - }); - }); - + try { - await downloadFile(tmpResult, description,url, eventStream, proxy, strictSSL); + await downloadFile(fd, description, url, eventStream, provider); eventStream.post(new DownloadSuccess(` Done!`)); } catch (primaryUrlError) { @@ -62,7 +27,7 @@ async function downloadPackage(description: string, url: string, fallbackUrl: st if (fallbackUrl) { eventStream.post(new DownloadFallBack(fallbackUrl)); try { - await downloadFile(tmpFile, description, fallbackUrl, eventStream, proxy, strictSSL); + await downloadFile(fd, description, fallbackUrl, eventStream, provider); eventStream.post(new DownloadSuccess(' Done!')); } catch (fallbackUrlError) { @@ -75,9 +40,11 @@ async function downloadPackage(description: string, url: string, fallbackUrl: st } } -async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string, urlString: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { +async function downloadFile(fd: number, description: string, urlString: string, eventStream: EventStream, provider: NetworkSettingsProvider): Promise { const url = parseUrl(urlString); - + const networkSettings = provider(); + const proxy = networkSettings.proxy; + const strictSSL = networkSettings.strictSSL; const options: https.RequestOptions = { host: url.hostname, path: url.path, @@ -87,27 +54,25 @@ async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string }; return new Promise((resolve, reject) => { - if (!tmpFile || tmpFile.fd == 0) { - return reject(new PackageError("Temporary package file unavailable", pkg)); - } + let request = https.request(options, response => { if (response.statusCode === 301 || response.statusCode === 302) { // Redirect - download from new location - return resolve(downloadFile(tmpFile, description, response.headers.location, eventStream, proxy, strictSSL)); + return resolve(downloadFile(fd, description, response.headers.location, eventStream, provider)); } if (response.statusCode != 200) { // Download failed - print error message eventStream.post(new DownloadFailure(`failed (error code '${response.statusCode}')`)); - return reject(new PackageError(response.statusCode.toString(), description)); + return reject(new NestedError(response.statusCode.toString())); } // Downloading - hook up events let packageSize = parseInt(response.headers['content-length'], 10); let downloadedBytes = 0; let downloadPercentage = 0; - let tmpFile = fs.createWriteStream(null, { fd: tmpFile.fd }); + let tmpFile = fs.createWriteStream(null, { fd }); eventStream.post(new DownloadSizeObtained(packageSize)); @@ -127,7 +92,7 @@ async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string }); response.on('error', err => { - reject(new PackageError(`Reponse error: ${err.message || 'NONE'}`, description, err)); + reject(new NestedError(`Reponse error: ${err.message || 'NONE'}`, err)); }); // Begin piping data from the response to the package file @@ -136,7 +101,7 @@ async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string request.on('error', err => { console.log(err); - reject(new PackageError(`Request error: ${err.message || 'NONE'}`, description, err)); + reject(new NestedError(`Request error: ${err.message || 'NONE'}`, err)); }); // Execute the request diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts new file mode 100644 index 000000000..4288ee5a1 --- /dev/null +++ b/src/packageManager/PackageFilterer.ts @@ -0,0 +1,37 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Package, PackageError, doesPackageTestPathExist } from "./packages"; +import { PlatformInformation } from "../platform"; + +export async function filterPackages(packages: Package[], platformInfo: PlatformInformation) { + let platformPackages = await filterPlatformPackages(packages, platformInfo); + return filterAlreadyInstalledPackages(platformPackages); +} + +async function filterPlatformPackages(packages: Package[], platformInfo: PlatformInformation) { + if (packages) { + return packages.filter(pkg => { + if (pkg.architectures && pkg.architectures.indexOf(platformInfo.architecture) === -1) { + return false; + } + + if (pkg.platforms && pkg.platforms.indexOf(platformInfo.platform) === -1) { + return false; + } + + return true; + }); + } + else { + throw new PackageError("Package manifest does not exist."); + } +} + +async function filterAlreadyInstalledPackages(packages: Package[]) { + return packages.filter(async pkg => { + return !(await doesPackageTestPathExist(pkg)); //return true if the package doesnot exist + }); +} \ No newline at end of file diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts index dc8c92492..fa1f70fe1 100644 --- a/src/packageManager/PackageInstaller.ts +++ b/src/packageManager/PackageInstaller.ts @@ -3,107 +3,98 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package, PackageError, getPackageTestPath, getBaseInstallPath } from "./packages"; -import { EventStream } from "../EventStream"; -import { InstallationProgress } from "../omnisharp/loggingEvents"; import * as fs from 'fs'; import * as mkdirp from 'mkdirp'; import * as yauzl from 'yauzl'; import * as path from 'path'; +import * as util from '../common'; +import { EventStream } from "../EventStream"; +import { InstallationProgress } from "../omnisharp/loggingEvents"; +import { NestedError } from './packages'; -export class PackageInstaller { - constructor() { } - public InstallPackage(pkg: Package, eventStream: EventStream): Promise { - const installationStage = 'installPackages'; - if (!pkg.tmpFile) { - // Download of this package was skipped, so there is nothing to install - return Promise.resolve(); - } +export async function InstallPackage(fd: number, description: string, installPath: string, installTestPath: string, binaries: string[], eventStream: EventStream): Promise { + const installationStage = 'installPackages'; - resolvePackageBinaries(pkg); - eventStream.post(new InstallationProgress(installationStage, pkg.description)); - - return new Promise((resolve, baseReject) => { - const reject = (err: any) => { - // If anything goes wrong with unzip, make sure we delete the test path (if there is one) - // so we will retry again later - const testPath = getPackageTestPath(pkg); - if (testPath) { - fs.unlink(testPath, unlinkErr => { - baseReject(err); - }); - } else { - baseReject(err); - } - }; + //to do: do not resolve the package binaries here + resolvePackageBinaries(binaries, installPath); + eventStream.post(new InstallationProgress(installationStage, description)); - if (pkg.tmpFile.fd == 0) { - return reject(new PackageError('Downloaded file unavailable', pkg)); + return new Promise((resolve, reject) => { + //to do: there was a code to unlink the file here. look into that!!!! + if (fd == 0) { + return reject(new NestedError('Downloaded file unavailable')); + } + + yauzl.fromFd(fd, { lazyEntries: true }, (err, zipFile) => { + if (err) { + return reject(new NestedError('Immediate zip file error', err)); } - yauzl.fromFd(pkg.tmpFile.fd, { lazyEntries: true }, (err, zipFile) => { - if (err) { - return reject(new PackageError('Immediate zip file error', pkg, err)); - } + zipFile.readEntry(); - zipFile.readEntry(); + zipFile.on('entry', (entry: yauzl.Entry) => { + let absoluteEntryPath = path.resolve(getBaseInstallPath(installPath), entry.fileName); - zipFile.on('entry', (entry: yauzl.Entry) => { - let absoluteEntryPath = path.resolve(getBaseInstallPath(pkg), entry.fileName); + if (entry.fileName.endsWith('/')) { + // Directory - create it + mkdirp(absoluteEntryPath, { mode: 0o775 }, err => { + if (err) { + return reject(new NestedError('Error creating directory for zip directory entry:' + err.code || '', err)); + } - if (entry.fileName.endsWith('/')) { - // Directory - create it - mkdirp(absoluteEntryPath, { mode: 0o775 }, err => { - if (err) { - return reject(new PackageError('Error creating directory for zip directory entry:' + err.code || '', pkg, err)); - } - - zipFile.readEntry(); - }); - } - else { - // File - extract it - zipFile.openReadStream(entry, (err, readStream) => { + zipFile.readEntry(); + }); + } + else { + // File - extract it + zipFile.openReadStream(entry, (err, readStream) => { + if (err) { + return reject(new NestedError('Error reading zip stream', err)); + } + + mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => { if (err) { - return reject(new PackageError('Error reading zip stream', pkg, err)); + return reject({ message: "", error: err }); + //new PackageError('Error creating directory for zip file entry', pkg, err)); } - mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => { - if (err) { - return reject(new PackageError('Error creating directory for zip file entry', pkg, err)); - } + // Make sure executable files have correct permissions when extracted + let fileMode = binaries && binaries.indexOf(absoluteEntryPath) !== -1 + ? 0o755 + : 0o664; - // Make sure executable files have correct permissions when extracted - let fileMode = pkg.binaries && pkg.binaries.indexOf(absoluteEntryPath) !== -1 - ? 0o755 - : 0o664; - - readStream.pipe(fs.createWriteStream(absoluteEntryPath, { mode: fileMode })); - readStream.on('end', () => zipFile.readEntry()); - }); + readStream.pipe(fs.createWriteStream(absoluteEntryPath, { mode: fileMode })); + readStream.on('end', () => zipFile.readEntry()); }); - } - }); + }); + } + }); - zipFile.on('end', () => { - resolve(); - }); + zipFile.on('end', () => { + resolve(); + }); - zipFile.on('error', err => { - reject(new PackageError('Zip File Error:' + err.code || '', pkg, err)); - }); + zipFile.on('error', err => { + reject(new NestedError('Zip File Error:' + err.code || '', err)); }); - }).then(() => { - // Clean up temp file - pkg.tmpFile.removeCallback(); }); - } + }); } -function resolvePackageBinaries(pkg: Package) { +//to do: remove these functions from here into a separate component +function resolvePackageBinaries(binaries: string[], installPath: string) { // Convert relative binary paths to absolute - if (pkg.binaries) { - pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value)); + if (binaries) { + binaries = binaries.map(value => path.resolve(getBaseInstallPath(installPath), value)); + } +} + +export function getBaseInstallPath(installPath: string): string { + let basePath = util.getExtensionPath(); + if (installPath) { + basePath = path.join(basePath, installPath); } -} \ No newline at end of file + + return basePath; +} diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 88ee3841f..bd846e014 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -4,61 +4,42 @@ *--------------------------------------------------------------------------------------------*/ import { PlatformInformation } from "../platform"; -import { Package, PackageError, doesPackageTestPathExist } from './packages'; -import { PackageDownloader } from './PackageDownloader'; -import { PackageInstaller } from './PackageInstaller'; -import { vscode } from '../vscodeAdapter'; +import { Package, PackageError, NestedError } from './packages'; +import { DownloadPackage } from './PackageDownloader'; +import { InstallPackage } from './PackageInstaller'; import { EventStream } from '../EventStream'; - -export class PackageManager { - public constructor() { - } - - private async filterPackages(packages: Package[], platformInfo: PlatformInformation) { - let platformPackages = await filterPlatformPackages(packages, platformInfo); - return filterAlreadyInstalledPackages(platformPackages); - } - - public async DownloadAndInstallPackages(packages: Package[], vscode: vscode, platformInfo: PlatformInformation, eventStream: EventStream) { - let filteredPackages = await this.filterPackages(packages, platformInfo); - if (filteredPackages) { - let downloader = new PackageDownloader(); - let installer = new PackageInstaller(); - for (let pkg of filteredPackages) { - await downloader.DownloadPackage(pkg, vscode, eventStream); - // see into error handling as well - await installer.InstallPackage(pkg, eventStream); +import { NetworkSettingsProvider } from "../NetworkSettings"; +import { filterPackages } from "./PackageFilterer"; +import { TmpFileManager } from './TmpFIleManager'; + +//Package manager needs a list of packages to be filtered based on platformInfo then download and install them +export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { + let filteredPackages = await filterPackages(packages, platformInfo); + if (filteredPackages) { + for (let pkg of filteredPackages) { + let tmpFileManager: TmpFileManager; + try { + tmpFileManager = new TmpFileManager(); + let tmpFile = await tmpFileManager.GetTmpFile(); + await DownloadPackage(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); + await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.installTestPath, pkg.binaries, eventStream); } - } - } -} - -async function filterPlatformPackages(packages: Package[], platformInfo: PlatformInformation) { - if (packages) { - return packages.filter(pkg => { - if (pkg.architectures && pkg.architectures.indexOf(platformInfo.architecture) === -1) { - return false; + catch (error) { + if (error instanceof NestedError) { + throw new PackageError(error.message, pkg, error.err); + } + else { + throw error; + } } - - if (pkg.platforms && pkg.platforms.indexOf(platformInfo.platform) === -1) { - return false; + finally { + //clean the temporary file + if (tmpFileManager) { + await tmpFileManager.CleanUpTmpFile(); + } + tmpFileManager = null; } - - return true; - }); - } - else { - throw new PackageError("Package manifest does not exist."); + } } } -async function filterAlreadyInstalledPackages(packages: Package[]) { - return packages.filter(async pkg => { - if (await doesPackageTestPathExist(pkg)) { - return false; // do not download the already installed packages - } - else { - return true; - } - }); -} \ No newline at end of file diff --git a/src/packageManager/TmpFileManager.ts b/src/packageManager/TmpFileManager.ts new file mode 100644 index 000000000..1d16127c4 --- /dev/null +++ b/src/packageManager/TmpFileManager.ts @@ -0,0 +1,37 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as tmp from 'tmp'; +import { NestedError } from './packages'; + +export class TmpFileManager { + private tmpFile: tmp.SynchrounousResult; + constructor() { } + + private async createTmpFile() { + this.tmpFile = await new Promise((resolve, reject) => { + tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { + if (err) { + return reject(new NestedError('Error from tmp.file', err)); + } + if (fd == 0) { + return reject(new NestedError("Temporary package file unavailable")); + } + + resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); + }); + }); + } + + public async GetTmpFile() { + await this.createTmpFile(); + return this.tmpFile; + } + + public async CleanUpTmpFile() { + if (this.tmpFile) { + this.tmpFile.removeCallback(); + } + } +} \ No newline at end of file diff --git a/src/packageManager/packages.ts b/src/packageManager/packages.ts index 4ed9cd33d..172df629c 100644 --- a/src/packageManager/packages.ts +++ b/src/packageManager/packages.ts @@ -4,7 +4,6 @@ *--------------------------------------------------------------------------------------------*/ import * as path from 'path'; -import * as tmp from 'tmp'; import * as util from '../common'; export interface Package { @@ -15,38 +14,29 @@ export interface Package { platforms: string[]; architectures: string[]; binaries: string[]; - tmpFile: tmp.SynchrounousResult; platformId?: string; // Path to use to test if the package has already been installed installTestPath?: string; } -export class PackageError extends Error { +export class NestedError extends Error { + constructor(public message: string, public err: any = null) { + super(message); + } +} + +export class PackageError extends NestedError { // Do not put PII (personally identifiable information) in the 'message' field as it will be logged to telemetry constructor(public message: string, - public packageDescription: string = null, + public pkg: Package = null, public innerError: any = null) { - super(message); + super(message, innerError); } } /*export class PackageManager { - public constructor() { } - - public async DownloadPackages(packages: Package[], eventStream: EventStream, proxy: string, strictSSL: boolean): Promise { - let packagesToDownload = await this.GetPackages(packages); - for(let pkg of packagesToDownload){ - await maybeDownloadPackage(pkg, eventStream, proxy, strictSSL); - } - return packagesToDownload; - } - - public async InstallPackages(packages: Package[], eventStream: EventStream): Promise { - let packagesToInstall = await this.GetPackages(packages); - return util.buildPromiseChain(packagesToInstall, async pkg => installPackage(pkg, eventStream)); - } - + public constructor() { public async GetLatestVersionFromFile(eventStream: EventStream, proxy: string, strictSSL: boolean, filePackage: Package): Promise { @@ -67,14 +57,9 @@ export class PackageError extends Error { } }*/ -export function getBaseInstallPath(pkg: Package): string { - let basePath = util.getExtensionPath(); - if (pkg.installPath) { - basePath = path.join(basePath, pkg.installPath); - } - - return basePath; -} +/* +Reolve all the paths here +*/ export async function doesPackageTestPathExist(pkg: Package): Promise { const testPath = getPackageTestPath(pkg); diff --git a/src/packageManager/proxy.ts b/src/packageManager/proxy.ts index 2d579d4a9..c572da761 100644 --- a/src/packageManager/proxy.ts +++ b/src/packageManager/proxy.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Url, parse as parseUrl } from 'url'; -import { isBoolean } from './common'; +import { isBoolean } from '../common'; import HttpProxyAgent = require('http-proxy-agent'); import HttpsProxyAgent = require('https-proxy-agent'); From 9cc00167e9a164d20e903a61ca9853d5f18158ad Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:46:24 -0700 Subject: [PATCH 15/64] modification for network settings --- tasks/offlinePackagingTasks.ts | 11 +- test/featureTests/testAssets/testAssets.ts | 234 +++++++++++---------- 2 files changed, 123 insertions(+), 122 deletions(-) diff --git a/tasks/offlinePackagingTasks.ts b/tasks/offlinePackagingTasks.ts index 3fa782b5a..3f934588e 100644 --- a/tasks/offlinePackagingTasks.ts +++ b/tasks/offlinePackagingTasks.ts @@ -19,8 +19,10 @@ import { CsharpLoggerObserver } from '../src/observers/CsharpLoggerObserver'; import { EventStream } from '../src/EventStream'; import { getPackageJSON } from '../tasks/packageJson'; import { Logger } from '../src/logger'; -import { PackageManager, Package } from '../src/packages'; import { PlatformInformation } from '../src/platform'; +import { Package } from '../src/packageManager/packages'; +import { DownloadAndInstallPackages } from '../src/packageManager/PackageManager'; +import NetworkSettings from '../src/NetworkSettings'; gulp.task('vsix:offline:package', async () => { del.sync(vscodeignorePath); @@ -87,17 +89,14 @@ async function doOfflinePackage(platformInfo: PlatformInformation, packageName: // Install Tasks async function install(platformInfo: PlatformInformation, packageJSON: any) { - const packageManager = new PackageManager(platformInfo); let eventStream = new EventStream(); const logger = new Logger(message => process.stdout.write(message)); let stdoutObserver = new CsharpLoggerObserver(logger); eventStream.subscribe(stdoutObserver.post); const debuggerUtil = new debugUtil.CoreClrDebugUtil(path.resolve('.')); let runTimeDependencies = JSON.parse(JSON.stringify(packageJSON.runtimeDependencies)); - - let downloadedPackages = await packageManager.DownloadPackages(runTimeDependencies, eventStream, undefined, undefined); - await packageManager.InstallPackages(downloadedPackages, eventStream); - await util.touchInstallFile(util.InstallFileType.Lock); + let provider = () => new NetworkSettings(undefined, undefined); + await DownloadAndInstallPackages(runTimeDependencies, provider, platformInfo, eventStream); await debugUtil.CoreClrDebugUtil.writeEmptyFile(debuggerUtil.installCompleteFilePath()); } diff --git a/test/featureTests/testAssets/testAssets.ts b/test/featureTests/testAssets/testAssets.ts index 65af8004b..72c3921b8 100644 --- a/test/featureTests/testAssets/testAssets.ts +++ b/test/featureTests/testAssets/testAssets.ts @@ -9,6 +9,7 @@ import { PlatformInformation } from "../../../src/platform"; import { OmnisharpDownloader } from "../../../src/omnisharp/OmnisharpDownloader"; import { getFakeVsCode, getNullWorkspaceConfiguration } from "../../unitTests/testAssets/Fakes"; import { Uri } from "../../../src/vscodeAdapter"; +import NetworkSettings from "../../../src/NetworkSettings"; export function GetTestOmnisharpDownloader(sink: EventStream, platformInfo: PlatformInformation): OmnisharpDownloader { @@ -18,123 +19,124 @@ export function GetTestOmnisharpDownloader(sink: EventStream, platformInfo: Plat ...getNullWorkspaceConfiguration(), }; }; - return new OmnisharpDownloader(vscode, sink, testPackageJSON, platformInfo); + + return new OmnisharpDownloader(() => new NetworkSettings(undefined, undefined), sink, testPackageJSON, platformInfo); } //Since we need only the runtime dependencies of packageJSON for the downloader create a testPackageJSON //with just that -export let testPackageJSON = { - "runtimeDependencies": [ - { - "description": "OmniSharp for Windows (.NET 4.6 / x86)", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505823/5804b7d3b5eeb7e4ae812a7cff03bd52/omnisharp-win-x86-1.28.0.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-win-x86-1.28.0.zip", - "installPath": ".omnisharp", - "platforms": [ - "win32" - ], - "architectures": [ - "x86" - ], - "installTestPath": "./.omnisharp/OmniSharp.exe", - "platformId": "win-x86" - }, - { - "description": "OmniSharp for Windows (.NET 4.6 / x64)", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505821/c570a9e20dbf7172f79850babd058872/omnisharp-win-x64-1.28.0.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-win-x64-1.28.0.zip", - "installPath": ".omnisharp", - "platforms": [ - "win32" - ], - "architectures": [ - "x86_64" - ], - "installTestPath": "./.omnisharp/OmniSharp.exe", - "platformId": "win-x64" - }, - { - "description": "OmniSharp for OSX", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505818/6b99c6a86da3221919158ca0f36a3e45/omnisharp-osx-1.28.0.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-osx-1.28.0.zip", - "installPath": ".omnisharp", - "platforms": [ - "darwin" - ], - "binaries": [ - "./mono.osx", - "./run" - ], - "installTestPath": "./.omnisharp/mono.osx", - "platformId": "osx" - }, - { - "description": "OmniSharp for Linux (x86)", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505817/b710ec9c2bedc0cfdb57da82da166c47/omnisharp-linux-x86-1.28.0.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-linux-x86-1.28.0.zip", - "installPath": ".omnisharp", - "platforms": [ - "linux" - ], - "architectures": [ - "x86", - "i686" - ], - "binaries": [ - "./mono.linux-x86", - "./run" - ], - "installTestPath": "./.omnisharp/mono.linux-x86", - "platformId": "linux-x86" - }, - { - "description": "OmniSharp for Linux (x64)", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505485/3f8a10409240decebb8a3189429f3fdf/omnisharp-linux-x64-1.28.0.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-linux-x64-1.28.0.zip", - "installPath": ".omnisharp", - "platforms": [ - "linux" - ], - "architectures": [ - "x86_64" - ], - "binaries": [ - "./mono.linux-x86_64", - "./run" - ], - "installTestPath": "./.omnisharp/mono.linux-x86_64", - "platformId": "linux-x64" - }, - { - "description": "OmniSharp for Test OS(architecture)", - "url": "https://download.visualstudio.microsoft.com/download/pr/100505485/3f8a10409240decebb8a3189429f3fdf/omnisharp-os-architecture-version.zip", - "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-os-architecture-version.zip", - "installPath": ".omnisharp", - "platforms": [ - "platform1" - ], - "architectures": [ - "architecture" - ], - "binaries": [ - "./binary1", - "./binary2" - ], - "installTestPath": "./.omnisharp/binary", - "platformId": "os-architecture" - }, - { - "description": "Non omnisharp package without platformId", - "url": "https://download.visualstudio.microsoft.com/download/pr/100317420/a30d7e11bc435433d297adc824ee837f/coreclr-debug-win7-x64.zip", - "fallbackUrl": "https://vsdebugger.blob.core.windows.net/coreclr-debug-1-14-4/coreclr-debug-win7-x64.zip", - "installPath": ".debugger", - "platforms": [ - "win32" - ], - "architectures": [ - "x86_64" - ], - "installTestPath": "./.debugger/vsdbg-ui.exe" - } - ] - }; \ No newline at end of file +export let testPackageJSON = { + "runtimeDependencies": [ + { + "description": "OmniSharp for Windows (.NET 4.6 / x86)", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505823/5804b7d3b5eeb7e4ae812a7cff03bd52/omnisharp-win-x86-1.28.0.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-win-x86-1.28.0.zip", + "installPath": ".omnisharp", + "platforms": [ + "win32" + ], + "architectures": [ + "x86" + ], + "installTestPath": "./.omnisharp/OmniSharp.exe", + "platformId": "win-x86" + }, + { + "description": "OmniSharp for Windows (.NET 4.6 / x64)", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505821/c570a9e20dbf7172f79850babd058872/omnisharp-win-x64-1.28.0.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-win-x64-1.28.0.zip", + "installPath": ".omnisharp", + "platforms": [ + "win32" + ], + "architectures": [ + "x86_64" + ], + "installTestPath": "./.omnisharp/OmniSharp.exe", + "platformId": "win-x64" + }, + { + "description": "OmniSharp for OSX", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505818/6b99c6a86da3221919158ca0f36a3e45/omnisharp-osx-1.28.0.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-osx-1.28.0.zip", + "installPath": ".omnisharp", + "platforms": [ + "darwin" + ], + "binaries": [ + "./mono.osx", + "./run" + ], + "installTestPath": "./.omnisharp/mono.osx", + "platformId": "osx" + }, + { + "description": "OmniSharp for Linux (x86)", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505817/b710ec9c2bedc0cfdb57da82da166c47/omnisharp-linux-x86-1.28.0.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-linux-x86-1.28.0.zip", + "installPath": ".omnisharp", + "platforms": [ + "linux" + ], + "architectures": [ + "x86", + "i686" + ], + "binaries": [ + "./mono.linux-x86", + "./run" + ], + "installTestPath": "./.omnisharp/mono.linux-x86", + "platformId": "linux-x86" + }, + { + "description": "OmniSharp for Linux (x64)", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505485/3f8a10409240decebb8a3189429f3fdf/omnisharp-linux-x64-1.28.0.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-linux-x64-1.28.0.zip", + "installPath": ".omnisharp", + "platforms": [ + "linux" + ], + "architectures": [ + "x86_64" + ], + "binaries": [ + "./mono.linux-x86_64", + "./run" + ], + "installTestPath": "./.omnisharp/mono.linux-x86_64", + "platformId": "linux-x64" + }, + { + "description": "OmniSharp for Test OS(architecture)", + "url": "https://download.visualstudio.microsoft.com/download/pr/100505485/3f8a10409240decebb8a3189429f3fdf/omnisharp-os-architecture-version.zip", + "fallbackUrl": "https://omnisharpdownload.blob.core.windows.net/ext/omnisharp-os-architecture-version.zip", + "installPath": ".omnisharp", + "platforms": [ + "platform1" + ], + "architectures": [ + "architecture" + ], + "binaries": [ + "./binary1", + "./binary2" + ], + "installTestPath": "./.omnisharp/binary", + "platformId": "os-architecture" + }, + { + "description": "Non omnisharp package without platformId", + "url": "https://download.visualstudio.microsoft.com/download/pr/100317420/a30d7e11bc435433d297adc824ee837f/coreclr-debug-win7-x64.zip", + "fallbackUrl": "https://vsdebugger.blob.core.windows.net/coreclr-debug-1-14-4/coreclr-debug-win7-x64.zip", + "installPath": ".debugger", + "platforms": [ + "win32" + ], + "architectures": [ + "x86_64" + ], + "installTestPath": "./.debugger/vsdbg-ui.exe" + } + ] +}; \ No newline at end of file From 98e7e74f3efbb225e89ccbed560753c57e084216 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:46:46 -0700 Subject: [PATCH 16/64] Clean up the downloaders --- src/CSharpExtDownloader.ts | 33 ++++--------- src/omnisharp/OmnisharpDownloader.ts | 47 +++++++++++-------- src/omnisharp/OmnisharpPackageCreator.ts | 8 +--- .../OmnisharpPackageCreator.test.ts | 16 +------ 4 files changed, 38 insertions(+), 66 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 8d3878347..aefc0036b 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -3,48 +3,37 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as util from './common'; import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; -import { vscode } from './vscodeAdapter'; -import { PackageManager } from './packageManager/PackageManager'; +import { DownloadAndInstallPackages } from './packageManager/PackageManager'; import { Package } from './packageManager/packages'; +import { NetworkSettingsProvider } from './NetworkSettings'; /* * Class used to download the runtime dependencies of the C# Extension */ export class CSharpExtDownloader { - private packageManager: PackageManager; public constructor( - private vscode: vscode, + private provider: NetworkSettingsProvider, private eventStream: EventStream, private packageJSON: any, private platformInfo: PlatformInformation) { - this.packageManager = new PackageManager(); } public async installRuntimeDependencies(): Promise { this.eventStream.post(new PackageInstallation("C# dependencies")); - let installationStage = 'touchBeginFile'; let success = false; + let installationStage = ''; try { - await util.touchInstallFile(util.InstallFileType.Begin); - // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); - let runTimeDependencies = this.GetRunTimeDependenciesPackages(); - installationStage = 'downloadAndInstallPackages'; - await this.packageManager.DownloadAndInstallPackages(runTimeDependencies, this.vscode, this.platformInfo, this.eventStream); - - // We probably dont need the install.Lock thing now as we are directly testing the package test path - installationStage = 'touchLockFile'; - await util.touchInstallFile(util.InstallFileType.Lock); - + await DownloadAndInstallPackages(runTimeDependencies, this.provider, this.platformInfo, this.eventStream); + //To do: We need to resolve the package binaries and the paths success = true; this.eventStream.post(new InstallationSuccess()); } @@ -52,21 +41,15 @@ export class CSharpExtDownloader { this.eventStream.post(new InstallationFailure(installationStage, error)); } finally { - // We do this step at the end so that we clean up the begin file in the case that we hit above catch block - // Attach a an empty catch to this so that errors here do not propogate - try { - util.deleteInstallFile(util.InstallFileType.Begin); - } - catch (error) { } return success; } } - public GetRunTimeDependenciesPackages(): Package[] { + private GetRunTimeDependenciesPackages(): Package[] { if (this.packageJSON.runtimeDependencies) { return JSON.parse(JSON.stringify(this.packageJSON.runtimeDependencies)); } return null; - } + } } diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index ee6e96c9f..0890d8696 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -3,26 +3,24 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { GetPackagesFromVersion, GetVersionFilePackage } from './OmnisharpPackageCreator'; +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import { GetPackagesFromVersion } from './OmnisharpPackageCreator'; import { PlatformInformation } from '../platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; import { EventStream } from '../EventStream'; -import { vscode } from '../vscodeAdapter'; -import { PackageManager } from '../packageManager/PackageManager'; - +import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; +import { NetworkSettingsProvider } from '../NetworkSettings'; +import { DownloadPackage } from '../packageManager/PackageDownloader'; +import { TmpFileManager } from '../packageManager/TmpFileManager'; export class OmnisharpDownloader { - private proxy: string; - private strictSSL: boolean; - private packageManager: PackageManager; public constructor( - private vscode: vscode, + private provider: NetworkSettingsProvider, private eventStream: EventStream, private packageJSON: any, private platformInfo: PlatformInformation) { - - this.packageManager = new PackageManager(); } public async DownloadAndInstallOmnisharp(version: string, serverUrl: string, installPath: string) { @@ -31,13 +29,10 @@ export class OmnisharpDownloader { try { this.eventStream.post(new LogPlatformInfo(this.platformInfo)); - installationStage = 'getPackageInfo'; let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); - installationStage = 'downloadAndInstallPackages'; - await this.packageManager.DownloadAndInstallPackages(packages, this.vscode, this.platformInfo, this.eventStream); - + await DownloadAndInstallPackages(packages, this.provider, this.platformInfo, this.eventStream); this.eventStream.post(new InstallationSuccess()); } catch (error) { @@ -46,18 +41,30 @@ export class OmnisharpDownloader { } } - public async GetLatestVersion(serverUrl: string, latestVersionFileServerPath: string): Promise { + public async GetLatestVersion(serverUrl: string, latestVersionFileServerPath: string) { let installationStage = 'getLatestVersionInfoFile'; + let description = "Latest Omnisharp Version Information"; + let url = `${serverUrl}/${latestVersionFileServerPath}`; + let latestVersion: string; + let tmpFileManager = new TmpFileManager(); try { this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); - //The package manager needs a package format to download, hence we form a package for the latest version file - let filePackage = GetVersionFilePackage(serverUrl, latestVersionFileServerPath); - //Fetch the latest version information from the file - return await this.packageManager.GetLatestVersionFromFile(this.eventStream, this.proxy, this.strictSSL, filePackage); + let tmpFile = await tmpFileManager.GetTmpFile(); + latestVersion = await this.DownloadLatestVersionFile(tmpFile, description, url, ""); // no fallback url + return latestVersion; } catch (error) { this.eventStream.post(new InstallationFailure(installationStage, error)); throw error; } + finally { + tmpFileManager.CleanUpTmpFile(); + } + } + + //To do: This component will move in a separate file + private async DownloadLatestVersionFile(tmpFile: tmp.SynchrounousResult, description: string, url: string, fallbackUrl: string): Promise { + await DownloadPackage(tmpFile.fd, description, url, "", this.eventStream, this.provider); + return fs.readFileSync(tmpFile.name, 'utf8'); } -} +} \ No newline at end of file diff --git a/src/omnisharp/OmnisharpPackageCreator.ts b/src/omnisharp/OmnisharpPackageCreator.ts index 76774f113..87130add5 100644 --- a/src/omnisharp/OmnisharpPackageCreator.ts +++ b/src/omnisharp/OmnisharpPackageCreator.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package } from "../packages"; +import { Package } from "../packageManager/packages"; export function GetPackagesFromVersion(version: string, runTimeDependencies: Package[], serverUrl: string, installPath: string): Package[] { if (!version) { @@ -54,9 +54,3 @@ function GetPackage(inputPackage: Package, serverUrl: string, version: string, i return versionPackage; } -export function GetVersionFilePackage(serverUrl: string, pathInServer: string): Package { - return { - "description": "Latest version information file", - "url": `${serverUrl}/${pathInServer}` - }; -} diff --git a/test/featureTests/OmnisharpPackageCreator.test.ts b/test/featureTests/OmnisharpPackageCreator.test.ts index 269bc9150..3eee9c99a 100644 --- a/test/featureTests/OmnisharpPackageCreator.test.ts +++ b/test/featureTests/OmnisharpPackageCreator.test.ts @@ -4,9 +4,9 @@ *--------------------------------------------------------------------------------------------*/ import { assert, should, expect } from "chai"; -import { Package } from "../../src/packages"; -import { SetBinaryAndGetPackage, GetPackagesFromVersion, GetVersionFilePackage } from "../../src/omnisharp/OmnisharpPackageCreator"; +import { SetBinaryAndGetPackage, GetPackagesFromVersion } from "../../src/omnisharp/OmnisharpPackageCreator"; import { testPackageJSON } from "./testAssets/testAssets"; +import { Package } from "../../src/packageManager/packages"; suite("GetOmnisharpPackage : Output package depends on the input package and other input parameters like serverUrl", () => { @@ -195,16 +195,4 @@ suite('GetPackagesFromVersion : Gets the experimental omnisharp packages from a outPackages.length.should.equal(1); outPackages[0].platformId.should.equal("win-x64"); }); -}); - -suite('GetVersionFilePackage : Gives the package for the latest file download', () => { - test('Contains the expected description', () => { - let testPackage = GetVersionFilePackage("someUrl", "somePath"); - expect(testPackage.description).to.equal('Latest version information file'); - }); - - test('Contains the url based on serverUrl and the pathInServer', () => { - let testPackage = GetVersionFilePackage("someUrl", "somePath"); - expect(testPackage.url).to.equal('someUrl/somePath'); - }); }); \ No newline at end of file From e20a43497a7567e31390f847e8a45c2002ae0b19 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 13:55:46 -0700 Subject: [PATCH 17/64] launch.json --- .vscode/launch.json | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0c0cf168e..c2006f78b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,20 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Launch Extension", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--extensionDevelopmentPath=${workspaceRoot}" + ], + "stopOnEntry": false, + "sourceMaps": true, + "outFiles": [ + "${workspaceRoot}/out/src/**/*.js" + ] + }, { "type": "node", "request": "launch", @@ -16,20 +30,8 @@ ], "sourceMaps": true, "internalConsoleOptions": "openOnSessionStart", - "outFiles" :["${workspaceRoot}/out/test/**/*.js"] - }, - { - "name": "Launch Extension", - "type": "extensionHost", - "request": "launch", - "runtimeExecutable": "${execPath}", - "args": [ - "--extensionDevelopmentPath=${workspaceRoot}" - ], - "stopOnEntry": false, - "sourceMaps": true, "outFiles": [ - "${workspaceRoot}/out/src/**/*.js" + "${workspaceRoot}/out/test/**/*.js" ] }, { @@ -100,4 +102,4 @@ ] } ] -} +} \ No newline at end of file From 97e83ec598b3a268eea7a10f09e25caec42f3644 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 14:08:04 -0700 Subject: [PATCH 18/64] Clean up --- src/CSharpExtDownloader.ts | 13 +++++++++++-- src/packageManager/packages.ts | 26 -------------------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index aefc0036b..149eb834c 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import * as util from './common'; import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; @@ -28,19 +29,27 @@ export class CSharpExtDownloader { let installationStage = ''; try { + + await util.touchInstallFile(util.InstallFileType.Begin); // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); let runTimeDependencies = this.GetRunTimeDependenciesPackages(); installationStage = 'downloadAndInstallPackages'; await DownloadAndInstallPackages(runTimeDependencies, this.provider, this.platformInfo, this.eventStream); - //To do: We need to resolve the package binaries and the paths + installationStage = 'touchLockFile'; + await util.touchInstallFile(util.InstallFileType.Lock); success = true; + this.eventStream.post(new InstallationSuccess()); } catch (error) { this.eventStream.post(new InstallationFailure(installationStage, error)); } finally { + try { + util.deleteInstallFile(util.InstallFileType.Begin); + } + catch (error) { } return success; } } @@ -52,4 +61,4 @@ export class CSharpExtDownloader { return null; } -} +} \ No newline at end of file diff --git a/src/packageManager/packages.ts b/src/packageManager/packages.ts index 172df629c..ccf200005 100644 --- a/src/packageManager/packages.ts +++ b/src/packageManager/packages.ts @@ -35,32 +35,6 @@ export class PackageError extends NestedError { } } -/*export class PackageManager { - public constructor() { - - - public async GetLatestVersionFromFile(eventStream: EventStream, proxy: string, strictSSL: boolean, filePackage: Package): Promise { - try { - let latestVersion: string; - await maybeDownloadPackage(filePackage, eventStream, proxy, strictSSL); - if (filePackage.tmpFile) { - latestVersion = fs.readFileSync(filePackage.tmpFile.name, 'utf8'); - //Delete the temporary file created - filePackage.tmpFile.removeCallback(); - } - - return latestVersion; - } - catch (error) { - throw new Error(`Could not download the latest version file due to ${error.toString()}`); - } - } -}*/ - -/* -Reolve all the paths here -*/ - export async function doesPackageTestPathExist(pkg: Package): Promise { const testPath = getPackageTestPath(pkg); if (testPath) { From e7b520aa89bc8b0dfabf7cec901b6b0beddd707e Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 15:15:43 -0700 Subject: [PATCH 19/64] do not use filter --- src/packageManager/PackageFilterer.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index 4288ee5a1..c829640fa 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -7,11 +7,11 @@ import { Package, PackageError, doesPackageTestPathExist } from "./packages"; import { PlatformInformation } from "../platform"; export async function filterPackages(packages: Package[], platformInfo: PlatformInformation) { - let platformPackages = await filterPlatformPackages(packages, platformInfo); + let platformPackages = filterPlatformPackages(packages, platformInfo); return filterAlreadyInstalledPackages(platformPackages); } -async function filterPlatformPackages(packages: Package[], platformInfo: PlatformInformation) { +function filterPlatformPackages(packages: Package[], platformInfo: PlatformInformation) { if (packages) { return packages.filter(pkg => { if (pkg.architectures && pkg.architectures.indexOf(platformInfo.architecture) === -1) { @@ -31,7 +31,13 @@ async function filterPlatformPackages(packages: Package[], platformInfo: Platfor } async function filterAlreadyInstalledPackages(packages: Package[]) { - return packages.filter(async pkg => { - return !(await doesPackageTestPathExist(pkg)); //return true if the package doesnot exist - }); + let packagesToInstall = []; + for (let pkg of packages) { + let exists = await doesPackageTestPathExist(pkg); + if (!exists) { + packagesToInstall.push(pkg); + } + } + + return packagesToInstall; } \ No newline at end of file From c6bee5ada1784087a38fbb7b616a0433e4d877e5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 15:21:31 -0700 Subject: [PATCH 20/64] use filter async --- package-lock.json | 13 +++++++++++++ package.json | 1 + src/packageManager/PackageFilterer.ts | 14 +++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index fad7855da..b835fd836 100644 --- a/package-lock.json +++ b/package-lock.json @@ -695,6 +695,11 @@ "inherits": "2.0.3" } }, + "bluebird": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", + "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" + }, "body-parser": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", @@ -4902,6 +4907,14 @@ "integrity": "sha512-8eRaxn8u/4wN8tGkhlc2cgwwvOLMLUMUn4IYTexMgWd+LyUDfeXVkk2ygQR0hvIHbJQXgHujia3ieUUDwNGkEA==", "dev": true }, + "node-filter-async": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/node-filter-async/-/node-filter-async-0.0.4.tgz", + "integrity": "sha512-W3p4yGNzH1822Z+CHitXY0uCy3vd9Vic7lEZv8NXHqT9erfod+UuZ/wN57oJwCqiSMoBqke62suyORwVb/mYIA==", + "requires": { + "bluebird": "3.5.1" + } + }, "node.extend": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/node.extend/-/node.extend-1.1.6.tgz", diff --git a/package.json b/package.json index 3e368cc0f..4788cc189 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "https-proxy-agent": "^2.1.1", "jsonc-parser": "^1.0.0", "mkdirp": "^0.5.1", + "node-filter-async": "0.0.4", "open": "*", "request-light": "^0.2.0", "rx": "^4.1.0", diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index c829640fa..c9777ed8c 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -6,6 +6,8 @@ import { Package, PackageError, doesPackageTestPathExist } from "./packages"; import { PlatformInformation } from "../platform"; +const { filterAsync } = require('node-filter-async'); + export async function filterPackages(packages: Package[], platformInfo: PlatformInformation) { let platformPackages = filterPlatformPackages(packages, platformInfo); return filterAlreadyInstalledPackages(platformPackages); @@ -31,13 +33,7 @@ function filterPlatformPackages(packages: Package[], platformInfo: PlatformInfor } async function filterAlreadyInstalledPackages(packages: Package[]) { - let packagesToInstall = []; - for (let pkg of packages) { - let exists = await doesPackageTestPathExist(pkg); - if (!exists) { - packagesToInstall.push(pkg); - } - } - - return packagesToInstall; + return filterAsync(packages, async (pkg: Package) => { + return !(await doesPackageTestPathExist(pkg)); + }); } \ No newline at end of file From 2dac16cbae6ca24fd5441feae2951dca5d8e4723 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 16:22:34 -0700 Subject: [PATCH 21/64] Use tmp file interface --- src/omnisharp/OmnisharpDownloader.ts | 15 +++++------ src/packageManager/CreateTmpFile.ts | 33 +++++++++++++++++++++++++ src/packageManager/PackageManager.ts | 12 ++++----- src/packageManager/TmpFileManager.ts | 37 ---------------------------- 4 files changed, 46 insertions(+), 51 deletions(-) create mode 100644 src/packageManager/CreateTmpFile.ts delete mode 100644 src/packageManager/TmpFileManager.ts diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 0890d8696..ef4068edd 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -4,15 +4,14 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; -import * as tmp from 'tmp'; import { GetPackagesFromVersion } from './OmnisharpPackageCreator'; import { PlatformInformation } from '../platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; import { EventStream } from '../EventStream'; -import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; import { NetworkSettingsProvider } from '../NetworkSettings'; +import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; +import { createTmpFile, TmpFile } from '../packageManager/CreateTmpFile'; import { DownloadPackage } from '../packageManager/PackageDownloader'; -import { TmpFileManager } from '../packageManager/TmpFileManager'; export class OmnisharpDownloader { @@ -46,10 +45,10 @@ export class OmnisharpDownloader { let description = "Latest Omnisharp Version Information"; let url = `${serverUrl}/${latestVersionFileServerPath}`; let latestVersion: string; - let tmpFileManager = new TmpFileManager(); + let tmpFile: TmpFile; try { this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); - let tmpFile = await tmpFileManager.GetTmpFile(); + tmpFile = await createTmpFile(); latestVersion = await this.DownloadLatestVersionFile(tmpFile, description, url, ""); // no fallback url return latestVersion; } @@ -58,12 +57,14 @@ export class OmnisharpDownloader { throw error; } finally { - tmpFileManager.CleanUpTmpFile(); + if (tmpFile) { + tmpFile.dispose(); + } } } //To do: This component will move in a separate file - private async DownloadLatestVersionFile(tmpFile: tmp.SynchrounousResult, description: string, url: string, fallbackUrl: string): Promise { + private async DownloadLatestVersionFile(tmpFile: TmpFile, description: string, url: string, fallbackUrl: string): Promise { await DownloadPackage(tmpFile.fd, description, url, "", this.eventStream, this.provider); return fs.readFileSync(tmpFile.name, 'utf8'); } diff --git a/src/packageManager/CreateTmpFile.ts b/src/packageManager/CreateTmpFile.ts new file mode 100644 index 000000000..9dbeb0ff5 --- /dev/null +++ b/src/packageManager/CreateTmpFile.ts @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as tmp from 'tmp'; +import { NestedError } from './packages'; + +export async function createTmpFile(): Promise { + const tmpFile = await new Promise((resolve, reject) => { + tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { + if (err) { + return reject(new NestedError('Error from tmp.file', err)); + } + if (fd == 0) { + return reject(new NestedError("Temporary package file unavailable")); + } + + resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); + }); + }); + + return { + fd: tmpFile.fd, + name: tmpFile.name, + dispose: tmpFile.removeCallback + }; +} + +export interface TmpFile { + fd: number; + name: string; + dispose: () => void; +} \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index bd846e014..0384dc2ab 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -10,17 +10,16 @@ import { InstallPackage } from './PackageInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; -import { TmpFileManager } from './TmpFIleManager'; +import { createTmpFile, TmpFile } from "./CreateTmpFile"; //Package manager needs a list of packages to be filtered based on platformInfo then download and install them export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { let filteredPackages = await filterPackages(packages, platformInfo); + let tmpFile: TmpFile; if (filteredPackages) { for (let pkg of filteredPackages) { - let tmpFileManager: TmpFileManager; try { - tmpFileManager = new TmpFileManager(); - let tmpFile = await tmpFileManager.GetTmpFile(); + tmpFile = await createTmpFile(); await DownloadPackage(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.installTestPath, pkg.binaries, eventStream); } @@ -34,10 +33,9 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: } finally { //clean the temporary file - if (tmpFileManager) { - await tmpFileManager.CleanUpTmpFile(); + if (tmpFile) { + await tmpFile.dispose(); } - tmpFileManager = null; } } } diff --git a/src/packageManager/TmpFileManager.ts b/src/packageManager/TmpFileManager.ts deleted file mode 100644 index 1d16127c4..000000000 --- a/src/packageManager/TmpFileManager.ts +++ /dev/null @@ -1,37 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ -import * as tmp from 'tmp'; -import { NestedError } from './packages'; - -export class TmpFileManager { - private tmpFile: tmp.SynchrounousResult; - constructor() { } - - private async createTmpFile() { - this.tmpFile = await new Promise((resolve, reject) => { - tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { - if (err) { - return reject(new NestedError('Error from tmp.file', err)); - } - if (fd == 0) { - return reject(new NestedError("Temporary package file unavailable")); - } - - resolve({ name: path, fd: fd, removeCallback: cleanupCallback }); - }); - }); - } - - public async GetTmpFile() { - await this.createTmpFile(); - return this.tmpFile; - } - - public async CleanUpTmpFile() { - if (this.tmpFile) { - this.tmpFile.removeCallback(); - } - } -} \ No newline at end of file From 85d867fe49ed6071d65fe340cf5f2cb151802757 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 16:40:19 -0700 Subject: [PATCH 22/64] Use tmpfile interface --- src/{packageManager => }/CreateTmpFile.ts | 2 +- src/omnisharp/OmnisharpDownloader.ts | 2 +- src/packageManager/PackageManager.ts | 2 +- .../Packages/PackageDownloader.test.ts | 18 +++++++++--------- 4 files changed, 12 insertions(+), 12 deletions(-) rename src/{packageManager => }/CreateTmpFile.ts (95%) diff --git a/src/packageManager/CreateTmpFile.ts b/src/CreateTmpFile.ts similarity index 95% rename from src/packageManager/CreateTmpFile.ts rename to src/CreateTmpFile.ts index 9dbeb0ff5..779a059ef 100644 --- a/src/packageManager/CreateTmpFile.ts +++ b/src/CreateTmpFile.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as tmp from 'tmp'; -import { NestedError } from './packages'; +import { NestedError } from './packageManager/packages'; export async function createTmpFile(): Promise { const tmpFile = await new Promise((resolve, reject) => { diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index ef4068edd..d233a49e0 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -10,7 +10,7 @@ import { PackageInstallation, LogPlatformInfo, InstallationSuccess, Installation import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; -import { createTmpFile, TmpFile } from '../packageManager/CreateTmpFile'; +import { createTmpFile, TmpFile } from '../CreateTmpFile'; import { DownloadPackage } from '../packageManager/PackageDownloader'; export class OmnisharpDownloader { diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 0384dc2ab..4115ab3c1 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -10,7 +10,7 @@ import { InstallPackage } from './PackageInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; -import { createTmpFile, TmpFile } from "./CreateTmpFile"; +import { createTmpFile, TmpFile } from "../CreateTmpFile"; //Package manager needs a list of packages to be filtered based on platformInfo then download and install them export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 5ac092fd0..9bb13d4d6 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -10,6 +10,7 @@ import { rimraf } from 'async-file'; import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; +import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; let ServerMock = require("mock-http-server"); const chai = require("chai"); @@ -25,18 +26,17 @@ suite("PackageDownloader : The package is downloaded ", () => { cert: fs.readFileSync("test/unitTests/testAssets/public.pem") }); - let tmpFile: tmp.SynchrounousResult = null; + let tmpFile: TmpFile; const eventStream = new EventStream(); const serverUrl = "https://127.0.0.1:9002"; setup(function (done) { server.start(done); - tmpFile = tmp.fileSync(); - util.setExtensionPath(tmpFile.name); }); - teardown(function (done) { - server.stop(done); + setup(async () => { + tmpFile = await createTmpFile(); + util.setExtensionPath(tmpFile.name); }); test('Packages are downloaded from the specified server url and installed at the specified path', async () => { @@ -64,11 +64,11 @@ suite("PackageDownloader : The package is downloaded ", () => { expect(stats.size).to.not.equal(0); }); - teardown(async () => { + teardown(function (done) { + server.stop(done); + if (tmpFile) { - await rimraf(tmpFile.name); + tmpFile.dispose(); } - - tmpFile = null; }); }); From 57d4bff5447ca97e9080a7f6b0d900a943d2a0ea Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 18:15:05 -0700 Subject: [PATCH 23/64] Check for the event stream --- .../Packages/PackageDownloader.test.ts | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 9bb13d4d6..455f0810c 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -4,20 +4,19 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; -import * as tmp from 'tmp'; import * as util from '../../../src/common'; -import { rimraf } from 'async-file'; import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; +import { BaseEvent, DownloadStart } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); chai.use(require("chai-as-promised")); let expect = chai.expect; -suite("PackageDownloader : The package is downloaded ", () => { +suite("PackageDownloader", () => { let server = new ServerMock({ host: "127.0.0.1", port: 9001 }, { host: "localhost", @@ -29,6 +28,7 @@ suite("PackageDownloader : The package is downloaded ", () => { let tmpFile: TmpFile; const eventStream = new EventStream(); const serverUrl = "https://127.0.0.1:9002"; + const description = "Test file"; setup(function (done) { server.start(done); @@ -39,31 +39,45 @@ suite("PackageDownloader : The package is downloaded ", () => { util.setExtensionPath(tmpFile.name); }); - test('Packages are downloaded from the specified server url and installed at the specified path', async () => { + test('File is downloaded from the specified url', async () => { server.on({ method: 'GET', path: '/resource', reply: { status: 200, headers: { "content-type": "text/plain" }, - body: "something" + body: "Test content" } }); - let description = "Latest version information file"; let url = `${serverUrl}/resource`; - try { - await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); - } - catch (error) { - console.log(error); - } - + await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); const stats = fs.statSync(tmpFile.name); + let text = fs.readFileSync(tmpFile.name, "utf8"); + expect(text).to.be.equal("Test content"); expect(stats.size).to.not.equal(0); }); + test('Events is created when the file is downloaded successfully', async () => { + server.on({ + method: 'GET', + path: '/resource', + reply: { + status: 200, + headers: { "content-type": "text/plain" }, + body: "Test content" + } + }); + + let url = `${serverUrl}/resource`; + let eventBus: BaseEvent[] = []; + eventStream.subscribe((event) => eventBus.push(event)); + await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); + console.log(eventBus); + //expect(eventBus).to.include([new DownloadStart('Test file')]); + }); + teardown(function (done) { server.stop(done); From 9afa848df452369e40fe655cff9a76281989bbb9 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 13 Apr 2018 18:29:09 -0700 Subject: [PATCH 24/64] package json --- package-lock.json | 2 +- test/unitTests/Packages/PackageDownloader.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index ccec9966f..09b1bbeee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8488,7 +8488,7 @@ "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" }, "pascalcase": { "version": "0.1.1", diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 455f0810c..e920b97e5 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -9,7 +9,7 @@ import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; -import { BaseEvent, DownloadStart } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); From e5b0d65db34998f819c322933977d830abe83bcb Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 10:56:59 -0700 Subject: [PATCH 25/64] Changes --- package-lock.json | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 09b1bbeee..8ea38e0da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -649,7 +649,6 @@ "requires": { "bytes": "3.0.0", "content-type": "1.0.4", - "debug": "2.6.9", "depd": "1.1.2", "http-errors": "1.6.3", "iconv-lite": "0.4.19", @@ -1090,7 +1089,6 @@ "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=", "dev": true, "requires": { - "debug": "2.6.9", "finalhandler": "1.1.0", "parseurl": "1.3.2", "utils-merge": "1.0.1" @@ -2020,7 +2018,6 @@ "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", "dev": true, "requires": { - "debug": "2.6.9", "encodeurl": "1.0.2", "escape-html": "1.0.3", "on-finished": "2.3.0", @@ -8488,7 +8485,8 @@ "parseurl": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=", + "dev": true }, "pascalcase": { "version": "0.1.1", @@ -9560,12 +9558,6 @@ "integrity": "sha1-5sgLYjEj19gM8TLOU480YokHJQI=", "dev": true }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", - "dev": true - }, "static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", @@ -9587,6 +9579,12 @@ } } }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "dev": true + }, "stream-combiner": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", From 7e77a6e4434a3944768fca0747e0be50cfdfc47b Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 12:13:13 -0700 Subject: [PATCH 26/64] Using FilePathResolver --- src/CSharpExtDownloader.ts | 2 + src/omnisharp/OmnisharpDownloader.ts | 2 + src/packageManager/PackageFilePathResolver.ts | 44 +++++++++++++++++++ src/packageManager/PackageFilterer.ts | 6 ++- src/packageManager/PackageInstaller.ts | 20 +-------- src/packageManager/PackageManager.ts | 1 + src/packageManager/packages.ts | 21 --------- .../Packages/PackageDownloader.test.ts | 11 +++-- 8 files changed, 61 insertions(+), 46 deletions(-) create mode 100644 src/packageManager/PackageFilePathResolver.ts diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 149eb834c..581cf105e 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -10,6 +10,7 @@ import { EventStream } from './EventStream'; import { DownloadAndInstallPackages } from './packageManager/PackageManager'; import { Package } from './packageManager/packages'; import { NetworkSettingsProvider } from './NetworkSettings'; +import { ResolveFilePaths } from './packageManager/PackageFilePathResolver'; /* * Class used to download the runtime dependencies of the C# Extension @@ -34,6 +35,7 @@ export class CSharpExtDownloader { // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); let runTimeDependencies = this.GetRunTimeDependenciesPackages(); + runTimeDependencies.forEach(pkg => ResolveFilePaths(pkg)); installationStage = 'downloadAndInstallPackages'; await DownloadAndInstallPackages(runTimeDependencies, this.provider, this.platformInfo, this.eventStream); installationStage = 'touchLockFile'; diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index d233a49e0..10c67f739 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -12,6 +12,7 @@ import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; import { createTmpFile, TmpFile } from '../CreateTmpFile'; import { DownloadPackage } from '../packageManager/PackageDownloader'; +import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; export class OmnisharpDownloader { @@ -30,6 +31,7 @@ export class OmnisharpDownloader { this.eventStream.post(new LogPlatformInfo(this.platformInfo)); installationStage = 'getPackageInfo'; let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); + packages.forEach(pkg => ResolveFilePaths(pkg)); installationStage = 'downloadAndInstallPackages'; await DownloadAndInstallPackages(packages, this.provider, this.platformInfo, this.eventStream); this.eventStream.post(new InstallationSuccess()); diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts new file mode 100644 index 000000000..9014abbe4 --- /dev/null +++ b/src/packageManager/PackageFilePathResolver.ts @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as path from 'path'; +import * as util from '../common'; +import { Package } from './packages'; + +export async function ResolveFilePaths(pkg: Package) { + pkg.installTestPath = ResolvePackageTestPath(pkg); + pkg.installPath = ResolveBaseInstallPath(pkg); + pkg.binaries = ResolvePackageBinaries(pkg); +} + +function ResolvePackageTestPath(pkg: Package): string { + if (path.isAbsolute(pkg.installTestPath)) { + return pkg.installTestPath; + } + + if (pkg.installTestPath) { + pkg.installTestPath = path.join(util.getExtensionPath(), pkg.installTestPath); + } + else { + return null; + } +} + +function ResolvePackageBinaries(pkg: Package) { + return pkg.binaries.map(value => path.resolve(ResolveBaseInstallPath(pkg), value)); +} + +function ResolveBaseInstallPath(pkg: Package): string { + if (path.isAbsolute(pkg.installPath)) { + return pkg.installPath; + } + + let basePath = util.getExtensionPath(); + if (pkg.installPath) { + basePath = path.join(basePath, pkg.installPath); + } + + return basePath; +} diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index c9777ed8c..2d7e120d6 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -3,8 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package, PackageError, doesPackageTestPathExist } from "./packages"; +import { Package, PackageError } from "./packages"; import { PlatformInformation } from "../platform"; +import * as util from '../common'; const { filterAsync } = require('node-filter-async'); @@ -34,6 +35,7 @@ function filterPlatformPackages(packages: Package[], platformInfo: PlatformInfor async function filterAlreadyInstalledPackages(packages: Package[]) { return filterAsync(packages, async (pkg: Package) => { - return !(await doesPackageTestPathExist(pkg)); + //If the file is present at the install test path then filter it + return !(await util.fileExists(pkg.installTestPath)); }); } \ No newline at end of file diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts index fa1f70fe1..f7ee7bdb2 100644 --- a/src/packageManager/PackageInstaller.ts +++ b/src/packageManager/PackageInstaller.ts @@ -16,8 +16,6 @@ import { NestedError } from './packages'; export async function InstallPackage(fd: number, description: string, installPath: string, installTestPath: string, binaries: string[], eventStream: EventStream): Promise { const installationStage = 'installPackages'; - //to do: do not resolve the package binaries here - resolvePackageBinaries(binaries, installPath); eventStream.post(new InstallationProgress(installationStage, description)); return new Promise((resolve, reject) => { @@ -34,7 +32,7 @@ export async function InstallPackage(fd: number, description: string, installPat zipFile.readEntry(); zipFile.on('entry', (entry: yauzl.Entry) => { - let absoluteEntryPath = path.resolve(getBaseInstallPath(installPath), entry.fileName); + let absoluteEntryPath = path.resolve(installPath, entry.fileName); if (entry.fileName.endsWith('/')) { // Directory - create it @@ -82,19 +80,3 @@ export async function InstallPackage(fd: number, description: string, installPat }); } -//to do: remove these functions from here into a separate component -function resolvePackageBinaries(binaries: string[], installPath: string) { - // Convert relative binary paths to absolute - if (binaries) { - binaries = binaries.map(value => path.resolve(getBaseInstallPath(installPath), value)); - } -} - -export function getBaseInstallPath(installPath: string): string { - let basePath = util.getExtensionPath(); - if (installPath) { - basePath = path.join(basePath, installPath); - } - - return basePath; -} diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 4115ab3c1..6a136f371 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -13,6 +13,7 @@ import { filterPackages } from "./PackageFilterer"; import { createTmpFile, TmpFile } from "../CreateTmpFile"; //Package manager needs a list of packages to be filtered based on platformInfo then download and install them +//Note that the packages that this component will install needs absolute paths for the installPath, intsallTestPath and the binaries export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { let filteredPackages = await filterPackages(packages, platformInfo); let tmpFile: TmpFile; diff --git a/src/packageManager/packages.ts b/src/packageManager/packages.ts index ccf200005..c57f106e1 100644 --- a/src/packageManager/packages.ts +++ b/src/packageManager/packages.ts @@ -3,9 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as path from 'path'; -import * as util from '../common'; - export interface Package { description: string; url: string; @@ -35,22 +32,4 @@ export class PackageError extends NestedError { } } -export async function doesPackageTestPathExist(pkg: Package): Promise { - const testPath = getPackageTestPath(pkg); - if (testPath) { - return util.fileExists(testPath); - } - else { - return Promise.resolve(false); - } -} - -export function getPackageTestPath(pkg: Package): string { - if (pkg.installTestPath) { - return path.join(util.getExtensionPath(), pkg.installTestPath); - } - else { - return null; - } -} diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index e920b97e5..385fcc1f6 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -9,11 +9,12 @@ import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; -import { BaseEvent } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); chai.use(require("chai-as-promised")); +chai.use(require('chai-arrays')); let expect = chai.expect; suite("PackageDownloader", () => { @@ -54,9 +55,10 @@ suite("PackageDownloader", () => { await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); const stats = fs.statSync(tmpFile.name); + expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); expect(text).to.be.equal("Test content"); - expect(stats.size).to.not.equal(0); + }); test('Events is created when the file is downloaded successfully', async () => { @@ -74,8 +76,9 @@ suite("PackageDownloader", () => { let eventBus: BaseEvent[] = []; eventStream.subscribe((event) => eventBus.push(event)); await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); - console.log(eventBus); - //expect(eventBus).to.include([new DownloadStart('Test file')]); + let eventNames = eventBus.map(elem => elem.constructor.name); + //Check whether these events appear in the expected order + expect(eventNames).to.have.ordered.members([DownloadStart.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name]); }); teardown(function (done) { From 76e9780ade6e3107d84c531a5e81e12c2c9f1059 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 12:23:34 -0700 Subject: [PATCH 27/64] Remove using --- src/packageManager/PackageInstaller.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts index f7ee7bdb2..af8ef3816 100644 --- a/src/packageManager/PackageInstaller.ts +++ b/src/packageManager/PackageInstaller.ts @@ -7,7 +7,6 @@ import * as fs from 'fs'; import * as mkdirp from 'mkdirp'; import * as yauzl from 'yauzl'; import * as path from 'path'; -import * as util from '../common'; import { EventStream } from "../EventStream"; import { InstallationProgress } from "../omnisharp/loggingEvents"; import { NestedError } from './packages'; From 47a18d1881a712fdd1461768bf385f287b3adb9a Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 14:14:25 -0700 Subject: [PATCH 28/64] Testing for normal and fallback case working --- src/packageManager/PackageDownloader.ts | 3 - src/packageManager/packages.ts | 1 - .../Packages/PackageDownloader.test.ts | 97 +++++++++++++------ 3 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/packageManager/PackageDownloader.ts b/src/packageManager/PackageDownloader.ts index 540d8dfae..6f72de699 100644 --- a/src/packageManager/PackageDownloader.ts +++ b/src/packageManager/PackageDownloader.ts @@ -54,8 +54,6 @@ async function downloadFile(fd: number, description: string, urlString: string, }; return new Promise((resolve, reject) => { - - let request = https.request(options, response => { if (response.statusCode === 301 || response.statusCode === 302) { // Redirect - download from new location @@ -100,7 +98,6 @@ async function downloadFile(fd: number, description: string, urlString: string, }); request.on('error', err => { - console.log(err); reject(new NestedError(`Request error: ${err.message || 'NONE'}`, err)); }); diff --git a/src/packageManager/packages.ts b/src/packageManager/packages.ts index c57f106e1..1ab260df9 100644 --- a/src/packageManager/packages.ts +++ b/src/packageManager/packages.ts @@ -12,7 +12,6 @@ export interface Package { architectures: string[]; binaries: string[]; platformId?: string; - // Path to use to test if the package has already been installed installTestPath?: string; } diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 385fcc1f6..46b0fcfc0 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -9,7 +9,7 @@ import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; -import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); @@ -18,18 +18,17 @@ chai.use(require('chai-arrays')); let expect = chai.expect; suite("PackageDownloader", () => { - let server = new ServerMock({ host: "127.0.0.1", port: 9001 }, + let server = new ServerMock(null, { host: "localhost", port: 9002, key: fs.readFileSync("test/unitTests/testAssets/private.pem"), cert: fs.readFileSync("test/unitTests/testAssets/public.pem") }); - + let tmpFile: TmpFile; const eventStream = new EventStream(); - const serverUrl = "https://127.0.0.1:9002"; - const description = "Test file"; + const fileDescription = "Test file"; setup(function (done) { server.start(done); @@ -40,8 +39,8 @@ suite("PackageDownloader", () => { util.setExtensionPath(tmpFile.name); }); - test('File is downloaded from the specified url', async () => { - server.on({ + suite('Response status Code is 200', () => { + const requestHandlerOptions = { method: 'GET', path: '/resource', reply: { @@ -49,37 +48,79 @@ suite("PackageDownloader", () => { headers: { "content-type": "text/plain" }, body: "Test content" } + }; + [ + { + description: "Download succeeds from the primary url", + url: "https://127.0.0.1:9002/resource", + fallBackUrl: "", + eventsSequence: [DownloadStart.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name] + }, + { + description: "Download succeeds from the fallback url", + url: "", + fallBackUrl: "https://127.0.0.1:9002/resource", + eventsSequence: [DownloadStart.name, DownloadFallBack.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name] + } + ].forEach((elem) => { + suite(elem.description, () => { + test('File is downloaded', async () => { + server.on(requestHandlerOptions); + await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, () => new NetworkSettings(undefined, false)); + const stats = fs.statSync(tmpFile.name); + expect(stats.size).to.not.equal(0); + let text = fs.readFileSync(tmpFile.name, "utf8"); + expect(text).to.be.equal("Test content"); + }); + + test('Events are created in the correct order', async () => { + server.on(requestHandlerOptions); + let eventBus: BaseEvent[] = []; + eventStream.subscribe((event) => eventBus.push(event)); + await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, () => new NetworkSettings(undefined, false)); + let eventNames = eventBus.map(elem => elem.constructor.name); + //Check whether these events appear in the expected order + expect(eventNames).to.have.ordered.members(elem.eventsSequence); + }); + }); }); - - let url = `${serverUrl}/resource`; - - await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); - const stats = fs.statSync(tmpFile.name); - expect(stats.size).to.not.equal(0); - let text = fs.readFileSync(tmpFile.name, "utf8"); - expect(text).to.be.equal("Test content"); - }); - test('Events is created when the file is downloaded successfully', async () => { - server.on({ + /*suite('Response Status Code is 301', () => { + const url = "https://127.0.0.1:9002/resource"; + const requestHandlerOptions = { + method: 'GET', + path: '/getResource', + reply: { + status: 301, + headers: { + "location": url + }, + } + }; + + const requestHandlerOptionsRedirect = { method: 'GET', path: '/resource', reply: { status: 200, - headers: { "content-type": "text/plain" }, + headers: { + "content-type": "text/plain", + }, body: "Test content" } - }); + }; - let url = `${serverUrl}/resource`; - let eventBus: BaseEvent[] = []; - eventStream.subscribe((event) => eventBus.push(event)); - await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); - let eventNames = eventBus.map(elem => elem.constructor.name); - //Check whether these events appear in the expected order - expect(eventNames).to.have.ordered.members([DownloadStart.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name]); - }); + test('File is downloaded from the redirect url', async () => { + server.on(requestHandlerOptions); + server.on(requestHandlerOptionsRedirect); + await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); + const stats = fs.statSync(tmpFile.name); + expect(stats.size).to.not.equal(0); + let text = fs.readFileSync(tmpFile.name, "utf8"); + expect(text).to.be.equal("Test content"); + }); + });*/ teardown(function (done) { server.stop(done); From 441153aaccbe3fb0b59b50d25498e054d99def88 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 15:01:47 -0700 Subject: [PATCH 29/64] Resolve the paths --- src/packageManager/PackageFilePathResolver.ts | 10 +++-- .../Packages/PackageDownloader.test.ts | 43 ++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts index 9014abbe4..54a775702 100644 --- a/src/packageManager/PackageFilePathResolver.ts +++ b/src/packageManager/PackageFilePathResolver.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import * as util from '../common'; import { Package } from './packages'; -export async function ResolveFilePaths(pkg: Package) { +export function ResolveFilePaths(pkg: Package) { pkg.installTestPath = ResolvePackageTestPath(pkg); pkg.installPath = ResolveBaseInstallPath(pkg); pkg.binaries = ResolvePackageBinaries(pkg); @@ -19,7 +19,7 @@ function ResolvePackageTestPath(pkg: Package): string { } if (pkg.installTestPath) { - pkg.installTestPath = path.join(util.getExtensionPath(), pkg.installTestPath); + return path.join(util.getExtensionPath(), pkg.installTestPath); } else { return null; @@ -27,7 +27,11 @@ function ResolvePackageTestPath(pkg: Package): string { } function ResolvePackageBinaries(pkg: Package) { - return pkg.binaries.map(value => path.resolve(ResolveBaseInstallPath(pkg), value)); + if (pkg.binaries) { + return pkg.binaries.map(value => path.resolve(ResolveBaseInstallPath(pkg), value)); + } + + return null; } function ResolveBaseInstallPath(pkg: Package): string { diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 46b0fcfc0..61587a915 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -9,7 +9,7 @@ import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; -import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); @@ -25,10 +25,13 @@ suite("PackageDownloader", () => { key: fs.readFileSync("test/unitTests/testAssets/private.pem"), cert: fs.readFileSync("test/unitTests/testAssets/public.pem") }); - + let tmpFile: TmpFile; const eventStream = new EventStream(); const fileDescription = "Test file"; + const networkSettingsProvider = () => new NetworkSettings(undefined, false); + let eventBus: BaseEvent[]; + eventStream.subscribe((event) => eventBus.push(event)); setup(function (done) { server.start(done); @@ -37,6 +40,7 @@ suite("PackageDownloader", () => { setup(async () => { tmpFile = await createTmpFile(); util.setExtensionPath(tmpFile.name); + eventBus = []; }); suite('Response status Code is 200', () => { @@ -53,7 +57,7 @@ suite("PackageDownloader", () => { { description: "Download succeeds from the primary url", url: "https://127.0.0.1:9002/resource", - fallBackUrl: "", + fallBackUrl: "", eventsSequence: [DownloadStart.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name] }, { @@ -66,18 +70,17 @@ suite("PackageDownloader", () => { suite(elem.description, () => { test('File is downloaded', async () => { server.on(requestHandlerOptions); - await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, () => new NetworkSettings(undefined, false)); + await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); expect(text).to.be.equal("Test content"); }); - + test('Events are created in the correct order', async () => { server.on(requestHandlerOptions); - let eventBus: BaseEvent[] = []; - eventStream.subscribe((event) => eventBus.push(event)); - await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, () => new NetworkSettings(undefined, false)); + + await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); let eventNames = eventBus.map(elem => elem.constructor.name); //Check whether these events appear in the expected order expect(eventNames).to.have.ordered.members(elem.eventsSequence); @@ -86,7 +89,7 @@ suite("PackageDownloader", () => { }); }); - /*suite('Response Status Code is 301', () => { + suite('Response Status Code is 301', () => { const url = "https://127.0.0.1:9002/resource"; const requestHandlerOptions = { method: 'GET', @@ -114,13 +117,31 @@ suite("PackageDownloader", () => { test('File is downloaded from the redirect url', async () => { server.on(requestHandlerOptions); server.on(requestHandlerOptionsRedirect); - await DownloadPackage(tmpFile.fd, description, url, "", eventStream, () => new NetworkSettings(undefined, false)); + await DownloadPackage(tmpFile.fd, fileDescription, "https://127.0.0.1:9002/getResource", "", eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); expect(text).to.be.equal("Test content"); }); - });*/ + }); + + suite('Response status code is not 301, 302 or 200', () => { + const url = "https://127.0.0.1:9002/resource"; + const requestHandlerOptions = { + method: 'GET', + path: '/resource', + reply: { + status: 404 + } + }; + + test('Error is thrown when the download fails', async () => { + server.on(requestHandlerOptions); + expect(DownloadPackage(tmpFile.fd, fileDescription, url, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); + let eventNames = eventBus.map(elem => elem.constructor.name); + expect(eventNames).to.be.containing(DownloadFailure.name); + }); + }); teardown(function (done) { server.stop(done); From df2f87441d76ffd593c981819430e0a074f2e5ff Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Mon, 16 Apr 2018 15:18:37 -0700 Subject: [PATCH 30/64] Remove failing case --- test/unitTests/Packages/PackageDownloader.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 61587a915..bf3062669 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -9,7 +9,7 @@ import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; -import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); const chai = require("chai"); @@ -138,8 +138,8 @@ suite("PackageDownloader", () => { test('Error is thrown when the download fails', async () => { server.on(requestHandlerOptions); expect(DownloadPackage(tmpFile.fd, fileDescription, url, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); - let eventNames = eventBus.map(elem => elem.constructor.name); - expect(eventNames).to.be.containing(DownloadFailure.name); + //let eventNames = eventBus.map(elem => elem.constructor.name); + //expect(eventNames).to.be.containing(DownloadFailure.name); }); }); From 4c2c15c92b9d4ceea46a618901e4ca650d9247bd Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 16 Apr 2018 18:36:41 -0700 Subject: [PATCH 31/64] package installer test-1 --- .../Packages/PackageInstaller.test.ts | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 test/unitTests/Packages/PackageInstaller.test.ts diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts new file mode 100644 index 000000000..69d78ba34 --- /dev/null +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -0,0 +1,81 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from 'fs'; +//import * as tmp from 'tmp'; +import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; +//import { InstallPackage } from '../../../src/packageManager/PackageInstaller'; +//import { EventStream } from '../../../src/EventStream'; +let archiver = require('archiver'); + +suite('PackageInstaller', () => { + let tmpSourceDir: TmpAsset; + //let tmpInstallDir: TmpAsset; + + const filesToAdd = [ + { + content: "File 1", + path: "file1.txt" + }, + { + content: "File 2", + path: "folder/file2.txt" + } + ]; + + const binariesToAdd = [ + { + content: "Binary 1", + path: "binary1.txt" + }, + ]; + + //let eventStream = new EventStream(); + + setup(() => { + //when cleaning, clean the directory even if it is not empty + tmpSourceDir = createTmpDir(true); + //tmpInstallDir = createTmpDir(true); + }); + + test('The package is installed', () => { + let testDirPath = tmpSourceDir.name + "test.zip"; + createTestZip(testDirPath, [...filesToAdd, ...binariesToAdd]); + //let fd = fs.openSync(testDirPath, 'r'); + //console.log(fs.readlinkSync('/proc/self/fd/' + fd)); + //console.log(testDirPath); + //await InstallPackage(tmpSourceDir.fd, "somePackage", tmpInstallDir.name, [], eventStream); + }); + + teardown(async () => { + tmpSourceDir.dispose(); + }); +}); + +function createTestZip(dirPath: string, filesToAdd: Array<{ content: string, path: string }>) { + + let output = fs.createWriteStream(dirPath); + let archive = archiver('zip'); + + archive.on('warning', function (err: any) { + if (err.code === 'ENOENT') { + console.log(err); + } else { + // throw error + throw err; + } + }); + + // good practice to catch this error explicitly + archive.on('error', function (err: any) { + throw err; + }); + + archive.pipe(output); + + filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); + // append a file from string + archive.finalize(); +} From 7e44a067309252f554f4689fbcb45168fc1e55cc Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 11:04:09 -0700 Subject: [PATCH 32/64] Add package installer test --- package-lock.json | 393 ++++++++++++++++++ package.json | 2 + src/{CreateTmpFile.ts => CreateTmpAsset.ts} | 31 +- src/omnisharp/OmnisharpDownloader.ts | 10 +- src/packageManager/PackageInstaller.ts | 6 +- src/packageManager/PackageManager.ts | 6 +- .../Packages/PackageDownloader.test.ts | 5 +- .../Packages/PackageInstaller.test.ts | 117 ++++-- 8 files changed, 511 insertions(+), 59 deletions(-) rename src/{CreateTmpFile.ts => CreateTmpAsset.ts} (61%) diff --git a/package-lock.json b/package-lock.json index 8ea38e0da..de94b9ac3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,15 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/archiver": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-2.1.1.tgz", + "integrity": "sha512-xejo0g2dIEy6qrQfOfOXNWwbBn397hTl8DHe6pETGwjjTCTdRER4j6vpaeWOBSMuC7zdFbp7i/iipGbo35IQEw==", + "dev": true, + "requires": { + "@types/glob": "5.0.35" + } + }, "@types/chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.2.tgz", @@ -301,6 +310,121 @@ "zone.js": "0.7.6" } }, + "archiver": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-2.1.1.tgz", + "integrity": "sha1-/2YrSnggFJSj7lRNOjP+dJZQnrw=", + "dev": true, + "requires": { + "archiver-utils": "1.3.0", + "async": "2.6.0", + "buffer-crc32": "0.2.13", + "glob": "7.1.2", + "lodash": "4.17.5", + "readable-stream": "2.3.6", + "tar-stream": "1.5.5", + "zip-stream": "1.2.0" + }, + "dependencies": { + "async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", + "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", + "dev": true, + "requires": { + "lodash": "4.17.5" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, + "archiver-utils": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-1.3.0.tgz", + "integrity": "sha1-5QtMCccL89aA4y/xt5lOn52JUXQ=", + "dev": true, + "requires": { + "glob": "7.1.2", + "graceful-fs": "4.1.11", + "lazystream": "1.0.0", + "lodash": "4.17.5", + "normalize-path": "2.1.1", + "readable-stream": "2.3.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", @@ -627,6 +751,48 @@ "array-events": "0.2.0" } }, + "bl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "dev": true, + "requires": { + "readable-stream": "2.3.6", + "safe-buffer": "5.1.1" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "block-stream": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", @@ -649,6 +815,7 @@ "requires": { "bytes": "3.0.0", "content-type": "1.0.4", + "debug": "2.6.9", "depd": "1.1.2", "http-errors": "1.6.3", "iconv-lite": "0.4.19", @@ -658,6 +825,15 @@ "type-is": "1.6.16" }, "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, "qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", @@ -1077,6 +1253,50 @@ "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=", "dev": true }, + "compress-commons": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-1.2.2.tgz", + "integrity": "sha1-UkqfEJA/OoEzibAiXSfEi7dRiQ8=", + "dev": true, + "requires": { + "buffer-crc32": "0.2.13", + "crc32-stream": "2.0.0", + "normalize-path": "2.1.1", + "readable-stream": "2.3.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1089,9 +1309,21 @@ "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=", "dev": true, "requires": { + "debug": "2.6.9", "finalhandler": "1.1.0", "parseurl": "1.3.2", "utils-merge": "1.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + } } }, "content-type": { @@ -1138,6 +1370,54 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true }, + "crc": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.5.0.tgz", + "integrity": "sha1-mLi6fUiWZbo5efWbITgTdBAaGWQ=", + "dev": true + }, + "crc32-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-2.0.0.tgz", + "integrity": "sha1-483TtN8xaN10494/u8t7KX/pCPQ=", + "dev": true, + "requires": { + "crc": "3.5.0", + "readable-stream": "2.3.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "cross-env": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-5.1.4.tgz", @@ -2018,6 +2298,7 @@ "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", "dev": true, "requires": { + "debug": "2.6.9", "encodeurl": "1.0.2", "escape-html": "1.0.3", "on-finished": "2.3.0", @@ -2026,6 +2307,15 @@ "unpipe": "1.0.0" }, "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, "statuses": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", @@ -9768,6 +10058,59 @@ } } }, + "tar-stream": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.5.tgz", + "integrity": "sha512-mQdgLPc/Vjfr3VWqWbfxW8yQNiJCbAZ+Gf6GDu1Cy0bdb33ofyiNGBtAY96jHFhDuivCwgW1H9DgTON+INiXgg==", + "dev": true, + "requires": { + "bl": "1.2.2", + "end-of-stream": "1.4.1", + "readable-stream": "2.3.6", + "xtend": "4.0.1" + }, + "dependencies": { + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "dev": true, + "requires": { + "once": "1.4.0" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -11001,6 +11344,56 @@ "integrity": "sha1-5a2ryKz0CPY4X8dklWhMiOavaJo=", "dev": true }, + "zip-stream": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-1.2.0.tgz", + "integrity": "sha1-qLxF9MG0lpnGuQGYuqyqzbzUugQ=", + "dev": true, + "requires": { + "archiver-utils": "1.3.0", + "compress-commons": "1.2.2", + "lodash": "4.17.5", + "readable-stream": "2.3.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==", + "dev": true + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + } + } + }, "zone.js": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.7.6.tgz", diff --git a/package.json b/package.json index 35f493e70..6005cf712 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "yauzl": "^2.5.0" }, "devDependencies": { + "@types/archiver": "^2.1.1", "@types/chai": "^4.1.2", "@types/chai-arrays": "1.0.2", "@types/chai-as-promised": "^7.1.0", @@ -96,6 +97,7 @@ "@types/node": "^9.4.7", "@types/semver": "^5.5.0", "@types/tmp": "0.0.33", + "archiver": "^2.1.1", "async-child-process": "^1.1.1", "async-file": "^2.0.2", "async-shelljs": "^0.1.2", diff --git a/src/CreateTmpFile.ts b/src/CreateTmpAsset.ts similarity index 61% rename from src/CreateTmpFile.ts rename to src/CreateTmpAsset.ts index 779a059ef..0665bb270 100644 --- a/src/CreateTmpFile.ts +++ b/src/CreateTmpAsset.ts @@ -4,8 +4,9 @@ *--------------------------------------------------------------------------------------------*/ import * as tmp from 'tmp'; import { NestedError } from './packageManager/packages'; +import { rimraf } from 'async-file'; -export async function createTmpFile(): Promise { +export async function createTmpFile(): Promise { const tmpFile = await new Promise((resolve, reject) => { tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { if (err) { @@ -22,11 +23,33 @@ export async function createTmpFile(): Promise { return { fd: tmpFile.fd, name: tmpFile.name, - dispose: tmpFile.removeCallback - }; + dispose: () => { + if (tmpFile) { + tmpFile.removeCallback(); + } + } + }; } -export interface TmpFile { +export function createTmpDir(unsafeCleanup: boolean): TmpAsset { + let tmpDir = tmp.dirSync(); + return { + fd: tmpDir.fd, + name: tmpDir.name, + dispose: async () => { + if (tmpDir) { + if (unsafeCleanup) { + await rimraf(tmpDir.name); + } + else { + tmpDir.removeCallback(); + } + } + } + }; +} + +export interface TmpAsset { fd: number; name: string; dispose: () => void; diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 10c67f739..13f970e1f 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -10,7 +10,7 @@ import { PackageInstallation, LogPlatformInfo, InstallationSuccess, Installation import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; -import { createTmpFile, TmpFile } from '../CreateTmpFile'; +import { createTmpFile, TmpAsset } from '../CreateTmpAsset'; import { DownloadPackage } from '../packageManager/PackageDownloader'; import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; @@ -47,7 +47,7 @@ export class OmnisharpDownloader { let description = "Latest Omnisharp Version Information"; let url = `${serverUrl}/${latestVersionFileServerPath}`; let latestVersion: string; - let tmpFile: TmpFile; + let tmpFile: TmpAsset; try { this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); tmpFile = await createTmpFile(); @@ -59,14 +59,12 @@ export class OmnisharpDownloader { throw error; } finally { - if (tmpFile) { - tmpFile.dispose(); - } + tmpFile.dispose(); } } //To do: This component will move in a separate file - private async DownloadLatestVersionFile(tmpFile: TmpFile, description: string, url: string, fallbackUrl: string): Promise { + private async DownloadLatestVersionFile(tmpFile: TmpAsset, description: string, url: string, fallbackUrl: string): Promise { await DownloadPackage(tmpFile.fd, description, url, "", this.eventStream, this.provider); return fs.readFileSync(tmpFile.name, 'utf8'); } diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts index af8ef3816..6d2893d5e 100644 --- a/src/packageManager/PackageInstaller.ts +++ b/src/packageManager/PackageInstaller.ts @@ -12,13 +12,12 @@ import { InstallationProgress } from "../omnisharp/loggingEvents"; import { NestedError } from './packages'; -export async function InstallPackage(fd: number, description: string, installPath: string, installTestPath: string, binaries: string[], eventStream: EventStream): Promise { +export async function InstallPackage(fd: number, description: string, installPath: string, binaries: string[], eventStream: EventStream): Promise { const installationStage = 'installPackages'; eventStream.post(new InstallationProgress(installationStage, description)); return new Promise((resolve, reject) => { - //to do: there was a code to unlink the file here. look into that!!!! if (fd == 0) { return reject(new NestedError('Downloaded file unavailable')); } @@ -52,8 +51,7 @@ export async function InstallPackage(fd: number, description: string, installPat mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => { if (err) { - return reject({ message: "", error: err }); - //new PackageError('Error creating directory for zip file entry', pkg, err)); + return reject(new NestedError('Error creating directory for zip file entry', err)); } // Make sure executable files have correct permissions when extracted diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 6a136f371..60b4cebb9 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -10,19 +10,19 @@ import { InstallPackage } from './PackageInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; -import { createTmpFile, TmpFile } from "../CreateTmpFile"; +import { createTmpFile, TmpAsset } from "../CreateTmpAsset"; //Package manager needs a list of packages to be filtered based on platformInfo then download and install them //Note that the packages that this component will install needs absolute paths for the installPath, intsallTestPath and the binaries export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { let filteredPackages = await filterPackages(packages, platformInfo); - let tmpFile: TmpFile; + let tmpFile: TmpAsset; if (filteredPackages) { for (let pkg of filteredPackages) { try { tmpFile = await createTmpFile(); await DownloadPackage(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); - await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.installTestPath, pkg.binaries, eventStream); + await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); } catch (error) { if (error instanceof NestedError) { diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index bf3062669..be7d13a76 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -8,7 +8,7 @@ import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; -import { TmpFile, createTmpFile } from '../../../src/CreateTmpFile'; +import { TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); @@ -26,7 +26,7 @@ suite("PackageDownloader", () => { cert: fs.readFileSync("test/unitTests/testAssets/public.pem") }); - let tmpFile: TmpFile; + let tmpFile: TmpAsset; const eventStream = new EventStream(); const fileDescription = "Test file"; const networkSettingsProvider = () => new NetworkSettings(undefined, false); @@ -79,7 +79,6 @@ suite("PackageDownloader", () => { test('Events are created in the correct order', async () => { server.on(requestHandlerOptions); - await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); let eventNames = eventBus.map(elem => elem.constructor.name); //Check whether these events appear in the expected order diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts index 69d78ba34..dbef52bee 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -4,20 +4,31 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; + +import * as path from 'path'; //import * as tmp from 'tmp'; import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; -//import { InstallPackage } from '../../../src/packageManager/PackageInstaller'; -//import { EventStream } from '../../../src/EventStream'; +import * as util from '../../../src/common'; +import { InstallPackage } from '../../../src/packageManager/PackageInstaller'; +import { EventStream } from '../../../src/EventStream'; +import { PlatformInformation } from '../../../src/platform'; let archiver = require('archiver'); +const chai = require("chai"); +let expect = chai.expect; suite('PackageInstaller', () => { let tmpSourceDir: TmpAsset; - //let tmpInstallDir: TmpAsset; + let tmpInstallDir: TmpAsset; + let signalClose: () => void; + let fileStreamClosePromise: Promise; + let testDirPath: string; + let fd: number; - const filesToAdd = [ + const files = [ { content: "File 1", - path: "file1.txt" + path: "file1.txt", + }, { content: "File 2", @@ -25,57 +36,85 @@ suite('PackageInstaller', () => { } ]; - const binariesToAdd = [ + const binaries = [ { content: "Binary 1", path: "binary1.txt" }, ]; - //let eventStream = new EventStream(); + const eventStream = new EventStream(); + let allFiles: Array<{ content: string, path: string }>; - setup(() => { - //when cleaning, clean the directory even if it is not empty + setup(async () => { tmpSourceDir = createTmpDir(true); - //tmpInstallDir = createTmpDir(true); + tmpInstallDir = createTmpDir(true); + fileStreamClosePromise = new Promise(resolve => { + signalClose = () => { + resolve(); + }; + }); + + allFiles = [...files, ...binaries]; + + testDirPath = tmpSourceDir.name + "/test.zip"; + createTestZip(testDirPath, allFiles); + await fileStreamClosePromise; + fd = fs.openSync(path.resolve(testDirPath), 'r'); }); - test('The package is installed', () => { - let testDirPath = tmpSourceDir.name + "test.zip"; - createTestZip(testDirPath, [...filesToAdd, ...binariesToAdd]); - //let fd = fs.openSync(testDirPath, 'r'); - //console.log(fs.readlinkSync('/proc/self/fd/' + fd)); - //console.log(testDirPath); - //await InstallPackage(tmpSourceDir.fd, "somePackage", tmpInstallDir.name, [], eventStream); + test('The folder is unzipped and all the files are present at the expected paths', async () => { + await InstallPackage(fd, "somePackage", tmpInstallDir.name, [], eventStream); + for (let elem of allFiles) { + let filePath = path.join(tmpInstallDir.name, elem.path); + expect(await util.fileExists(filePath)).to.be.true; + } + // check whether the files were installed correctly }); - teardown(async () => { - tmpSourceDir.dispose(); + test('The folder is unzipped and the binaries have the expected permissions on Linux', async () => { + if ((await PlatformInformation.GetCurrent()).isLinux()) { + await InstallPackage(fd, "somePackage", tmpInstallDir.name, binaries.map(binary => binary.path), eventStream); + for (let elem of binaries) { + let filePath = path.join(tmpInstallDir.name, elem.path); + expect(await util.fileExists(filePath)).to.be.true; + let mode = fs.statSync(filePath).mode; + expect(mode).to.be.equal(0o755); + } + } }); -}); -function createTestZip(dirPath: string, filesToAdd: Array<{ content: string, path: string }>) { - let output = fs.createWriteStream(dirPath); - let archive = archiver('zip'); - archive.on('warning', function (err: any) { - if (err.code === 'ENOENT') { - console.log(err); - } else { - // throw error - throw err; - } + teardown(async () => { + fs.closeSync(fd); + //Close the opened file + tmpSourceDir.dispose(); + tmpInstallDir.dispose(); + fileStreamClosePromise = undefined; }); - // good practice to catch this error explicitly - archive.on('error', function (err: any) { - throw err; - }); + function createTestZip(dirPath: string, filesToAdd: Array<{ content: string, path: string }>) { + let output = fs.createWriteStream(dirPath); + output.on('close', function () { + signalClose(); // the installer needs to wait for the filestream to be closed here + }); + let archive = archiver('zip'); + archive.on('warning', function (err: any) { + if (err.code === 'ENOENT') { + console.log(err); + } else { + // throw error + throw err; + } + }); - archive.pipe(output); + archive.on('error', function (err: any) { + throw err; + }); - filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); - // append a file from string - archive.finalize(); -} + archive.pipe(output); + filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); + archive.finalize(); + } +}); From e218cb4fe62c458595bc16be0c6379354ab1e425 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 13:48:33 -0700 Subject: [PATCH 33/64] Create tmp asset --- src/CreateTmpAsset.ts | 47 ++++++------ src/omnisharp/OmnisharpDownloader.ts | 6 +- .../Packages/PackageInstaller.test.ts | 75 ++++++++----------- 3 files changed, 59 insertions(+), 69 deletions(-) diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index 0665bb270..b304e4db1 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -23,34 +23,37 @@ export async function createTmpFile(): Promise { return { fd: tmpFile.fd, name: tmpFile.name, - dispose: () => { - if (tmpFile) { - tmpFile.removeCallback(); - } - } - }; + dispose: () => tmpFile.removeCallback() + }; } -export function createTmpDir(unsafeCleanup: boolean): TmpAsset { - let tmpDir = tmp.dirSync(); +export async function createTmpDir(unsafeCleanup: boolean): Promise { + const tmpDir = await new Promise((resolve, reject) => { + tmp.dir({ unsafeCleanup }, (err, path, cleanupCallback) => { + if (err) { + return reject(new NestedError('Error from tmp.file', err)); + } + + resolve({ name: path, removeCallback: cleanupCallback }); + }); + }); + return { fd: tmpDir.fd, name: tmpDir.name, - dispose: async () => { - if (tmpDir) { - if (unsafeCleanup) { - await rimraf(tmpDir.name); - } - else { - tmpDir.removeCallback(); - } + dispose: () => { + if (unsafeCleanup) { + rimraf(tmpDir.name);//to delete directories that have folders inside them } - } + else { + tmpDir.removeCallback(); + } + } }; -} +} export interface TmpAsset { - fd: number; - name: string; - dispose: () => void; -} \ No newline at end of file + fd?: number; + name: string; + dispose: () => void; + } \ No newline at end of file diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 13f970e1f..a9b109ecd 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -59,9 +59,11 @@ export class OmnisharpDownloader { throw error; } finally { - tmpFile.dispose(); + if (tmpFile) { + tmpFile.dispose(); + } } - } + } //To do: This component will move in a separate file private async DownloadLatestVersionFile(tmpFile: TmpAsset, description: string, url: string, fallbackUrl: string): Promise { diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts index dbef52bee..36d5dfdf4 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -3,8 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as fs from 'fs'; - +import * as fs from 'async-file'; import * as path from 'path'; //import * as tmp from 'tmp'; import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; @@ -12,15 +11,13 @@ import * as util from '../../../src/common'; import { InstallPackage } from '../../../src/packageManager/PackageInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; -let archiver = require('archiver'); +import * as archiver from 'archiver'; const chai = require("chai"); let expect = chai.expect; suite('PackageInstaller', () => { let tmpSourceDir: TmpAsset; let tmpInstallDir: TmpAsset; - let signalClose: () => void; - let fileStreamClosePromise: Promise; let testDirPath: string; let fd: number; @@ -47,20 +44,12 @@ suite('PackageInstaller', () => { let allFiles: Array<{ content: string, path: string }>; setup(async () => { - tmpSourceDir = createTmpDir(true); - tmpInstallDir = createTmpDir(true); - fileStreamClosePromise = new Promise(resolve => { - signalClose = () => { - resolve(); - }; - }); - + tmpSourceDir = await createTmpDir(true); + tmpInstallDir = await createTmpDir(true); allFiles = [...files, ...binaries]; - testDirPath = tmpSourceDir.name + "/test.zip"; - createTestZip(testDirPath, allFiles); - await fileStreamClosePromise; - fd = fs.openSync(path.resolve(testDirPath), 'r'); + await createTestZipAsync(testDirPath, allFiles); + fd = await fs.open(path.resolve(testDirPath), 'r'); }); test('The folder is unzipped and all the files are present at the expected paths', async () => { @@ -69,52 +58,48 @@ suite('PackageInstaller', () => { let filePath = path.join(tmpInstallDir.name, elem.path); expect(await util.fileExists(filePath)).to.be.true; } - // check whether the files were installed correctly }); - test('The folder is unzipped and the binaries have the expected permissions on Linux', async () => { - if ((await PlatformInformation.GetCurrent()).isLinux()) { + test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { + if (!(await PlatformInformation.GetCurrent()).isWindows()) { await InstallPackage(fd, "somePackage", tmpInstallDir.name, binaries.map(binary => binary.path), eventStream); for (let elem of binaries) { let filePath = path.join(tmpInstallDir.name, elem.path); expect(await util.fileExists(filePath)).to.be.true; - let mode = fs.statSync(filePath).mode; + let mode = (await fs.stat(filePath)).mode; expect(mode).to.be.equal(0o755); } } }); - - teardown(async () => { - fs.closeSync(fd); - //Close the opened file + await fs.close(fd); tmpSourceDir.dispose(); tmpInstallDir.dispose(); - fileStreamClosePromise = undefined; }); - function createTestZip(dirPath: string, filesToAdd: Array<{ content: string, path: string }>) { + function createTestZipAsync(dirPath: string, filesToAdd: Array<{ content: string, path: string }>): Promise { let output = fs.createWriteStream(dirPath); - output.on('close', function () { - signalClose(); // the installer needs to wait for the filestream to be closed here - }); - let archive = archiver('zip'); - archive.on('warning', function (err: any) { - if (err.code === 'ENOENT') { - console.log(err); - } else { - // throw error - throw err; - } - }); - archive.on('error', function (err: any) { - throw err; + return new Promise((resolve, reject) => { + output.on('close', function () { + resolve(); // the installer needs to wait for the filestream to be closed here + }); + + let archive = archiver('zip'); + archive.on('warning', function (err: any) { + if (err.code === 'ENOENT') { + console.log(err); + } else { + // throw error + reject(err); + } + }); + + archive.on('error', reject); + archive.pipe(output); + filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); + archive.finalize(); }); - - archive.pipe(output); - filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); - archive.finalize(); } }); From b96fb220a786f12fbc5efae482e504c571ba24f2 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 16:01:50 -0700 Subject: [PATCH 34/64] Package Downloader test refactored --- src/CreateTmpAsset.ts | 2 +- .../Packages/PackageDownloader.test.ts | 151 +++++++++--------- 2 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index b304e4db1..1def57ee2 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -53,7 +53,7 @@ export async function createTmpDir(unsafeCleanup: boolean): Promise { } export interface TmpAsset { - fd?: number; + fd: number; name: string; dispose: () => void; } \ No newline at end of file diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index be7d13a76..61eb00f56 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -4,24 +4,26 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; +import * as chai from 'chai'; import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; -import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); -const chai = require("chai"); chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); let expect = chai.expect; +//to do: fail on http +//to do: test for proxy suite("PackageDownloader", () => { - let server = new ServerMock(null, + let server = new ServerMock({ host: "localhost", port: 9000 }, { host: "localhost", - port: 9002, + port: 8080, key: fs.readFileSync("test/unitTests/testAssets/private.pem"), cert: fs.readFileSync("test/unitTests/testAssets/public.pem") }); @@ -32,44 +34,81 @@ suite("PackageDownloader", () => { const networkSettingsProvider = () => new NetworkSettings(undefined, false); let eventBus: BaseEvent[]; eventStream.subscribe((event) => eventBus.push(event)); + const httpsServerUrl = "https://127.0.0.1:8080"; + const correctUrlPath = `/resource`; + const redirectUrlPath = '/redirectResource'; + const errorUrlPath = '/errorResource'; + const correctUrl = `${httpsServerUrl}${correctUrlPath}`; + const redirectUrl = `${httpsServerUrl}${redirectUrlPath}`; + const errorUrl = `${httpsServerUrl}${errorUrlPath}`; - setup(function (done) { - server.start(done); - }); + const requestOptions = { + method: 'GET', + path: correctUrlPath, + reply: { + status: 200, + headers: { "content-type": "text/plain" }, + body: "Test content" + } + }; + + const requestOptionsRedirect = { + method: 'GET', + path: redirectUrlPath, + reply: { + status: 301, + headers: { + "location": correctUrl + }, + } + }; + + const requestOptionsError = { + method: 'GET', + path: errorUrlPath, + reply: { + status: 404, + } + }; setup(async () => { + await new Promise(resolve => server.start(resolve)); tmpFile = await createTmpFile(); util.setExtensionPath(tmpFile.name); eventBus = []; + server.on(requestOptions); + server.on(requestOptionsError); + server.on(requestOptionsRedirect); }); - suite('Response status Code is 200', () => { - const requestHandlerOptions = { - method: 'GET', - path: '/resource', - reply: { - status: 200, - headers: { "content-type": "text/plain" }, - body: "Test content" - } - }; + suite('If the response status Code is 200, the download succeeds', () => { + [ { - description: "Download succeeds from the primary url", - url: "https://127.0.0.1:9002/resource", + description: "Primary url", + url: correctUrl, fallBackUrl: "", - eventsSequence: [DownloadStart.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name] + eventsSequence: [ + new DownloadStart(fileDescription), + new DownloadSizeObtained(12), + new DownloadProgress(100, fileDescription), + new DownloadSuccess(' Done!')] }, { - description: "Download succeeds from the fallback url", - url: "", - fallBackUrl: "https://127.0.0.1:9002/resource", - eventsSequence: [DownloadStart.name, DownloadFallBack.name, DownloadSizeObtained.name, DownloadProgress.name, DownloadSuccess.name] + description: "Fallback url", + url: errorUrl, + fallBackUrl: correctUrl, + eventsSequence: [ + new DownloadStart(fileDescription), + new DownloadFailure("failed (error code '404')"), + new DownloadFallBack(correctUrl), + new DownloadSizeObtained(12), + new DownloadProgress(100, fileDescription), + new DownloadSuccess(' Done!')] } ].forEach((elem) => { suite(elem.description, () => { test('File is downloaded', async () => { - server.on(requestHandlerOptions); await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); @@ -78,45 +117,16 @@ suite("PackageDownloader", () => { }); test('Events are created in the correct order', async () => { - server.on(requestHandlerOptions); await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); - let eventNames = eventBus.map(elem => elem.constructor.name); - //Check whether these events appear in the expected order - expect(eventNames).to.have.ordered.members(elem.eventsSequence); + expect(eventBus).to.be.deep.equal(elem.eventsSequence); }); }); }); }); - suite('Response Status Code is 301', () => { - const url = "https://127.0.0.1:9002/resource"; - const requestHandlerOptions = { - method: 'GET', - path: '/getResource', - reply: { - status: 301, - headers: { - "location": url - }, - } - }; - - const requestHandlerOptionsRedirect = { - method: 'GET', - path: '/resource', - reply: { - status: 200, - headers: { - "content-type": "text/plain", - }, - body: "Test content" - } - }; - + suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { - server.on(requestHandlerOptions); - server.on(requestHandlerOptionsRedirect); - await DownloadPackage(tmpFile.fd, fileDescription, "https://127.0.0.1:9002/getResource", "", eventStream, networkSettingsProvider); + await DownloadPackage(tmpFile.fd, fileDescription, redirectUrl, "", eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); @@ -124,27 +134,22 @@ suite("PackageDownloader", () => { }); }); - suite('Response status code is not 301, 302 or 200', () => { - const url = "https://127.0.0.1:9002/resource"; - const requestHandlerOptions = { - method: 'GET', - path: '/resource', - reply: { - status: 404 - } - }; + suite('If the response status code is not 301, 302 or 200 then the download fails', () => { + test('Error is thrown', async () => { + expect(DownloadPackage(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); + }); + + test('Download Start and Download Failure events are created', () => { - test('Error is thrown when the download fails', async () => { - server.on(requestHandlerOptions); - expect(DownloadPackage(tmpFile.fd, fileDescription, url, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); - //let eventNames = eventBus.map(elem => elem.constructor.name); - //expect(eventNames).to.be.containing(DownloadFailure.name); }); }); - teardown(function (done) { - server.stop(done); + /*test('Fails if http url is provided', async () => { + await DownloadPackage(tmpFile.fd, fileDescription, "http://127.0.0.1/resource", "", eventStream, networkSettingsProvider); + });*/ + teardown(async () => { + await new Promise((resolve, reject) => server.stop(resolve)); if (tmpFile) { tmpFile.dispose(); } From 10b6073a86216326e01c8d2076d0d19cfed157b7 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 16:04:21 -0700 Subject: [PATCH 35/64] Rename files --- .../{PackageDownloader.ts => FileDownloader.ts} | 0 src/packageManager/PackageManager.ts | 4 ++-- src/packageManager/{PackageInstaller.ts => ZipInstaller.ts} | 0 test/unitTests/Packages/PackageDownloader.test.ts | 2 +- test/unitTests/Packages/PackageInstaller.test.ts | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/packageManager/{PackageDownloader.ts => FileDownloader.ts} (100%) rename src/packageManager/{PackageInstaller.ts => ZipInstaller.ts} (100%) diff --git a/src/packageManager/PackageDownloader.ts b/src/packageManager/FileDownloader.ts similarity index 100% rename from src/packageManager/PackageDownloader.ts rename to src/packageManager/FileDownloader.ts diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 60b4cebb9..0c0cf1e35 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -5,8 +5,8 @@ import { PlatformInformation } from "../platform"; import { Package, PackageError, NestedError } from './packages'; -import { DownloadPackage } from './PackageDownloader'; -import { InstallPackage } from './PackageInstaller'; +import { DownloadPackage } from './FileDownloader'; +import { InstallPackage } from './ZipInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/ZipInstaller.ts similarity index 100% rename from src/packageManager/PackageInstaller.ts rename to src/packageManager/ZipInstaller.ts diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/PackageDownloader.test.ts index 61eb00f56..1f9fe6c0f 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/PackageDownloader.test.ts @@ -7,7 +7,7 @@ import * as fs from 'fs'; import * as chai from 'chai'; import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; -import { DownloadPackage } from '../../../src/packageManager/PackageDownloader'; +import { DownloadPackage } from '../../../src/packageManager/FileDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts index 36d5dfdf4..2080caaaf 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -8,7 +8,7 @@ import * as path from 'path'; //import * as tmp from 'tmp'; import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; import * as util from '../../../src/common'; -import { InstallPackage } from '../../../src/packageManager/PackageInstaller'; +import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; import * as archiver from 'archiver'; From b9ac792f0d20ff1056490fcfbeda19286a7b9294 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 17 Apr 2018 16:13:10 -0700 Subject: [PATCH 36/64] resolve compile issues --- src/omnisharp/OmnisharpDownloader.ts | 2 +- test/unitTests/Packages/PackageInstaller.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index a9b109ecd..8f0680528 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -11,7 +11,7 @@ import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; import { createTmpFile, TmpAsset } from '../CreateTmpAsset'; -import { DownloadPackage } from '../packageManager/PackageDownloader'; +import { DownloadPackage } from '../packageManager/FileDownloader'; import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; export class OmnisharpDownloader { diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts index 2080caaaf..536e94b40 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -78,7 +78,7 @@ suite('PackageInstaller', () => { tmpInstallDir.dispose(); }); - function createTestZipAsync(dirPath: string, filesToAdd: Array<{ content: string, path: string }>): Promise { + async function createTestZipAsync(dirPath: string, filesToAdd: Array<{ content: string, path: string }>): Promise<{}> { let output = fs.createWriteStream(dirPath); return new Promise((resolve, reject) => { From 10c6811c7cd5c99ec8a95f723216db79cea0a5bd Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 17 Apr 2018 16:28:35 -0700 Subject: [PATCH 37/64] Clean up installer --- .../Packages/PackageInstaller.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/PackageInstaller.test.ts index 536e94b40..b22bdda1d 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/PackageInstaller.test.ts @@ -5,14 +5,14 @@ import * as fs from 'async-file'; import * as path from 'path'; -//import * as tmp from 'tmp'; -import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; +import * as chai from 'chai'; import * as util from '../../../src/common'; +import * as archiver from 'archiver'; +import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; -import * as archiver from 'archiver'; -const chai = require("chai"); + let expect = chai.expect; suite('PackageInstaller', () => { @@ -62,11 +62,11 @@ suite('PackageInstaller', () => { test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { if (!(await PlatformInformation.GetCurrent()).isWindows()) { - await InstallPackage(fd, "somePackage", tmpInstallDir.name, binaries.map(binary => binary.path), eventStream); - for (let elem of binaries) { - let filePath = path.join(tmpInstallDir.name, elem.path); - expect(await util.fileExists(filePath)).to.be.true; - let mode = (await fs.stat(filePath)).mode; + let resolvedBinaryPaths = binaries.map(binary => path.join(tmpInstallDir.name, binary.path)); + await InstallPackage(fd, "somePackage", tmpInstallDir.name, resolvedBinaryPaths, eventStream); + for (let binaryPath of resolvedBinaryPaths) { + expect(await util.fileExists(binaryPath)).to.be.true; + let mode = (await fs.stat(binaryPath)).mode; expect(mode).to.be.equal(0o755); } } From 053efe3ede2ab0964625e18338d4bcb7e0f435f8 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 17 Apr 2018 16:54:45 -0700 Subject: [PATCH 38/64] Clean up the tests --- ...nloader.test.ts => FileDownloader.test.ts} | 19 +++++++------ ...Installer.test.ts => ZipInstaller.test.ts} | 27 +++++++++++++++++-- 2 files changed, 36 insertions(+), 10 deletions(-) rename test/unitTests/Packages/{PackageDownloader.test.ts => FileDownloader.test.ts} (93%) rename test/unitTests/Packages/{PackageInstaller.test.ts => ZipInstaller.test.ts} (74%) diff --git a/test/unitTests/Packages/PackageDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts similarity index 93% rename from test/unitTests/Packages/PackageDownloader.test.ts rename to test/unitTests/Packages/FileDownloader.test.ts index 1f9fe6c0f..c3711346a 100644 --- a/test/unitTests/Packages/PackageDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -17,8 +17,6 @@ chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); let expect = chai.expect; -//to do: fail on http -//to do: test for proxy suite("PackageDownloader", () => { let server = new ServerMock({ host: "localhost", port: 9000 }, { @@ -139,15 +137,20 @@ suite("PackageDownloader", () => { expect(DownloadPackage(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); }); - test('Download Start and Download Failure events are created', () => { - + test('Download Start and Download Failure events are created', async () => { + let eventsSequence = [ + new DownloadStart(fileDescription), + new DownloadFailure("failed (error code '404')") + ]; + try { + await DownloadPackage(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider); + } + catch (error) { + expect(eventBus).to.be.deep.equal(eventsSequence); + } }); }); - /*test('Fails if http url is provided', async () => { - await DownloadPackage(tmpFile.fd, fileDescription, "http://127.0.0.1/resource", "", eventStream, networkSettingsProvider); - });*/ - teardown(async () => { await new Promise((resolve, reject) => server.stop(resolve)); if (tmpFile) { diff --git a/test/unitTests/Packages/PackageInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts similarity index 74% rename from test/unitTests/Packages/PackageInstaller.test.ts rename to test/unitTests/Packages/ZipInstaller.test.ts index b22bdda1d..bd912dd7c 100644 --- a/test/unitTests/Packages/PackageInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -12,7 +12,9 @@ import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; +import { BaseEvent, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; +chai.use(require("chai-as-promised")); let expect = chai.expect; suite('PackageInstaller', () => { @@ -40,10 +42,14 @@ suite('PackageInstaller', () => { }, ]; + const fileDescription = "somefile"; const eventStream = new EventStream(); + let eventBus: BaseEvent[]; + eventStream.subscribe((event) => eventBus.push(event)); let allFiles: Array<{ content: string, path: string }>; setup(async () => { + eventBus = []; tmpSourceDir = await createTmpDir(true); tmpInstallDir = await createTmpDir(true); allFiles = [...files, ...binaries]; @@ -53,17 +59,25 @@ suite('PackageInstaller', () => { }); test('The folder is unzipped and all the files are present at the expected paths', async () => { - await InstallPackage(fd, "somePackage", tmpInstallDir.name, [], eventStream); + await InstallPackage(fd, fileDescription, tmpInstallDir.name, [], eventStream); for (let elem of allFiles) { let filePath = path.join(tmpInstallDir.name, elem.path); expect(await util.fileExists(filePath)).to.be.true; } }); + test('The folder is unzipped and all the expected events are created', async () => { + await InstallPackage(fd, fileDescription, tmpInstallDir.name, [], eventStream); + let eventSequence: BaseEvent[] = [ + new InstallationProgress('installPackages', fileDescription) + ]; + expect(eventBus).to.be.deep.equal(eventSequence); + }); + test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { if (!(await PlatformInformation.GetCurrent()).isWindows()) { let resolvedBinaryPaths = binaries.map(binary => path.join(tmpInstallDir.name, binary.path)); - await InstallPackage(fd, "somePackage", tmpInstallDir.name, resolvedBinaryPaths, eventStream); + await InstallPackage(fd, fileDescription, tmpInstallDir.name, resolvedBinaryPaths, eventStream); for (let binaryPath of resolvedBinaryPaths) { expect(await util.fileExists(binaryPath)).to.be.true; let mode = (await fs.stat(binaryPath)).mode; @@ -72,6 +86,15 @@ suite('PackageInstaller', () => { } }); + test('Error is thrown on incorrect install path', async () => { + expect(InstallPackage(fd, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; + }); + + test('Error is thrown on invalid input file', async () => { + //providing a random file descriptor + expect(InstallPackage(fd + Math.random(), fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; + }); + teardown(async () => { await fs.close(fd); tmpSourceDir.dispose(); From 286fa7c6c9b8dfd9533e04b2f72c7dcb17d7a61a Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 17:21:42 -0700 Subject: [PATCH 39/64] Nits --- src/CSharpExtDownloader.ts | 1 - src/CreateTmpAsset.ts | 2 +- src/omnisharp/OmnisharpDownloader.ts | 13 +++---------- src/packageManager/FileDownloader.ts | 6 +++++- src/packageManager/PackageFilePathResolver.ts | 12 ++---------- src/packageManager/PackageFilterer.ts | 3 ++- src/packageManager/PackageManager.ts | 4 ++-- test/unitTests/Packages/FileDownloader.test.ts | 17 +++++++++++------ test/unitTests/Packages/ZipInstaller.test.ts | 4 ++-- 9 files changed, 28 insertions(+), 34 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 581cf105e..cefd62132 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -41,7 +41,6 @@ export class CSharpExtDownloader { installationStage = 'touchLockFile'; await util.touchInstallFile(util.InstallFileType.Lock); success = true; - this.eventStream.post(new InstallationSuccess()); } catch (error) { diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index 1def57ee2..313b34917 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -31,7 +31,7 @@ export async function createTmpDir(unsafeCleanup: boolean): Promise { const tmpDir = await new Promise((resolve, reject) => { tmp.dir({ unsafeCleanup }, (err, path, cleanupCallback) => { if (err) { - return reject(new NestedError('Error from tmp.file', err)); + return reject(new NestedError('Error from tmp.dir', err)); } resolve({ name: path, removeCallback: cleanupCallback }); diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 8f0680528..4cf0a3985 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -11,7 +11,7 @@ import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; import { createTmpFile, TmpAsset } from '../CreateTmpAsset'; -import { DownloadPackage } from '../packageManager/FileDownloader'; +import { DownloadFile } from '../packageManager/FileDownloader'; import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; export class OmnisharpDownloader { @@ -46,13 +46,12 @@ export class OmnisharpDownloader { let installationStage = 'getLatestVersionInfoFile'; let description = "Latest Omnisharp Version Information"; let url = `${serverUrl}/${latestVersionFileServerPath}`; - let latestVersion: string; let tmpFile: TmpAsset; try { this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); tmpFile = await createTmpFile(); - latestVersion = await this.DownloadLatestVersionFile(tmpFile, description, url, ""); // no fallback url - return latestVersion; + await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.provider); + return fs.readFileSync(tmpFile.name, 'utf8'); } catch (error) { this.eventStream.post(new InstallationFailure(installationStage, error)); @@ -63,11 +62,5 @@ export class OmnisharpDownloader { tmpFile.dispose(); } } - } - - //To do: This component will move in a separate file - private async DownloadLatestVersionFile(tmpFile: TmpAsset, description: string, url: string, fallbackUrl: string): Promise { - await DownloadPackage(tmpFile.fd, description, url, "", this.eventStream, this.provider); - return fs.readFileSync(tmpFile.name, 'utf8'); } } \ No newline at end of file diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index 6f72de699..bb3bc4f33 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -13,7 +13,7 @@ import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; -export async function DownloadPackage(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, provider: NetworkSettingsProvider){ +export async function DownloadFile(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, provider: NetworkSettingsProvider){ eventStream.post(new DownloadStart(description)); try { @@ -54,6 +54,10 @@ async function downloadFile(fd: number, description: string, urlString: string, }; return new Promise((resolve, reject) => { + if (fd == 0) { + reject(new NestedError("Temporary package file unavailable")); + } + let request = https.request(options, response => { if (response.statusCode === 301 || response.statusCode === 302) { // Redirect - download from new location diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts index 54a775702..64acbd793 100644 --- a/src/packageManager/PackageFilePathResolver.ts +++ b/src/packageManager/PackageFilePathResolver.ts @@ -13,13 +13,9 @@ export function ResolveFilePaths(pkg: Package) { pkg.binaries = ResolvePackageBinaries(pkg); } -function ResolvePackageTestPath(pkg: Package): string { - if (path.isAbsolute(pkg.installTestPath)) { - return pkg.installTestPath; - } - +export function ResolvePackageTestPath(pkg: Package): string { if (pkg.installTestPath) { - return path.join(util.getExtensionPath(), pkg.installTestPath); + return path.resolve(util.getExtensionPath(), pkg.installTestPath); } else { return null; @@ -35,10 +31,6 @@ function ResolvePackageBinaries(pkg: Package) { } function ResolveBaseInstallPath(pkg: Package): string { - if (path.isAbsolute(pkg.installPath)) { - return pkg.installPath; - } - let basePath = util.getExtensionPath(); if (pkg.installPath) { basePath = path.join(basePath, pkg.installPath); diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index 2d7e120d6..f821f41b6 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -6,6 +6,7 @@ import { Package, PackageError } from "./packages"; import { PlatformInformation } from "../platform"; import * as util from '../common'; +import { ResolvePackageTestPath } from "./PackageFilePathResolver"; const { filterAsync } = require('node-filter-async'); @@ -36,6 +37,6 @@ function filterPlatformPackages(packages: Package[], platformInfo: PlatformInfor async function filterAlreadyInstalledPackages(packages: Package[]) { return filterAsync(packages, async (pkg: Package) => { //If the file is present at the install test path then filter it - return !(await util.fileExists(pkg.installTestPath)); + return !(await util.fileExists(ResolvePackageTestPath(pkg))); }); } \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 0c0cf1e35..5c4b02877 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -5,7 +5,7 @@ import { PlatformInformation } from "../platform"; import { Package, PackageError, NestedError } from './packages'; -import { DownloadPackage } from './FileDownloader'; +import { DownloadFile } from './FileDownloader'; import { InstallPackage } from './ZipInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; @@ -21,7 +21,7 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: for (let pkg of filteredPackages) { try { tmpFile = await createTmpFile(); - await DownloadPackage(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); + await DownloadFile(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); } catch (error) { diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index c3711346a..b908d13a4 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -7,7 +7,7 @@ import * as fs from 'fs'; import * as chai from 'chai'; import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; -import { DownloadPackage } from '../../../src/packageManager/FileDownloader'; +import { DownloadFile } from '../../../src/packageManager/FileDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; @@ -107,7 +107,7 @@ suite("PackageDownloader", () => { ].forEach((elem) => { suite(elem.description, () => { test('File is downloaded', async () => { - await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); @@ -115,7 +115,7 @@ suite("PackageDownloader", () => { }); test('Events are created in the correct order', async () => { - await DownloadPackage(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); expect(eventBus).to.be.deep.equal(elem.eventsSequence); }); }); @@ -124,7 +124,7 @@ suite("PackageDownloader", () => { suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { - await DownloadPackage(tmpFile.fd, fileDescription, redirectUrl, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, redirectUrl, "", eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); @@ -134,7 +134,7 @@ suite("PackageDownloader", () => { suite('If the response status code is not 301, 302 or 200 then the download fails', () => { test('Error is thrown', async () => { - expect(DownloadPackage(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); + expect(DownloadFile(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); }); test('Download Start and Download Failure events are created', async () => { @@ -143,7 +143,7 @@ suite("PackageDownloader", () => { new DownloadFailure("failed (error code '404')") ]; try { - await DownloadPackage(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider); } catch (error) { expect(eventBus).to.be.deep.equal(eventsSequence); @@ -151,6 +151,11 @@ suite("PackageDownloader", () => { }); }); + test('Error is thrown on invalid input file', async () => { + //fd=0 means there is no file + expect(DownloadFile(0, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).to.be.rejected; + }); + teardown(async () => { await new Promise((resolve, reject) => server.stop(resolve)); if (tmpFile) { diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index bd912dd7c..6556c6659 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -91,8 +91,8 @@ suite('PackageInstaller', () => { }); test('Error is thrown on invalid input file', async () => { - //providing a random file descriptor - expect(InstallPackage(fd + Math.random(), fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; + //fd=0 means there is no file + expect(InstallPackage(0, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; }); teardown(async () => { From 12dfc3fa3228890d4955b6c1d0245a45df6a7439 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 17 Apr 2018 17:42:22 -0700 Subject: [PATCH 40/64] Rename packages --- src/CSharpExtDownloader.ts | 2 +- src/CreateTmpAsset.ts | 2 +- src/observers/CsharpLoggerObserver.ts | 2 +- src/observers/TelemetryObserver.ts | 2 +- src/omnisharp/OmnisharpPackageCreator.ts | 2 +- src/packageManager/FileDownloader.ts | 2 +- src/packageManager/{packages.ts => Package.ts} | 0 src/packageManager/PackageFilePathResolver.ts | 2 +- src/packageManager/PackageFilterer.ts | 2 +- src/packageManager/PackageManager.ts | 2 +- src/packageManager/ZipInstaller.ts | 2 +- src/tools/UpdatePackageDependencies.ts | 2 +- tasks/offlinePackagingTasks.ts | 2 +- test/featureTests/OmnisharpPackageCreator.test.ts | 2 +- test/unitTests/logging/CsharpLoggerObserver.test.ts | 2 +- test/unitTests/logging/TelemetryObserver.test.ts | 2 +- 16 files changed, 15 insertions(+), 15 deletions(-) rename src/packageManager/{packages.ts => Package.ts} (100%) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index cefd62132..0ef839a43 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -8,7 +8,7 @@ import { PlatformInformation } from './platform'; import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents'; import { EventStream } from './EventStream'; import { DownloadAndInstallPackages } from './packageManager/PackageManager'; -import { Package } from './packageManager/packages'; +import { Package } from './packageManager/Package'; import { NetworkSettingsProvider } from './NetworkSettings'; import { ResolveFilePaths } from './packageManager/PackageFilePathResolver'; diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index 313b34917..3a4a5edd1 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as tmp from 'tmp'; -import { NestedError } from './packageManager/packages'; +import { NestedError } from './packageManager/Package'; import { rimraf } from 'async-file'; export async function createTmpFile(): Promise { diff --git a/src/observers/CsharpLoggerObserver.ts b/src/observers/CsharpLoggerObserver.ts index 3105106ae..139ed6a84 100644 --- a/src/observers/CsharpLoggerObserver.ts +++ b/src/observers/CsharpLoggerObserver.ts @@ -5,7 +5,7 @@ import { BaseLoggerObserver } from "./BaseLoggerObserver"; import * as Event from "../omnisharp/loggingEvents"; -import { PackageError } from "../packageManager/packages"; +import { PackageError } from "../packageManager/Package"; export class CsharpLoggerObserver extends BaseLoggerObserver { private dots: number; diff --git a/src/observers/TelemetryObserver.ts b/src/observers/TelemetryObserver.ts index 5d1b6d30b..8911d10a7 100644 --- a/src/observers/TelemetryObserver.ts +++ b/src/observers/TelemetryObserver.ts @@ -5,7 +5,7 @@ import { PlatformInformation } from "../platform"; import { BaseEvent, PackageInstallation, InstallationFailure, InstallationSuccess, OmnisharpDelayTrackerEventMeasures, OmnisharpStart, TestExecutionCountReport, TelemetryEventWithMeasures } from "../omnisharp/loggingEvents"; -import { PackageError } from "../packageManager/packages"; +import { PackageError } from "../packageManager/Package"; export interface ITelemetryReporter { sendTelemetryEvent(eventName: string, properties?: { [key: string]: string }, measures?: { [key: string]: number }): void; diff --git a/src/omnisharp/OmnisharpPackageCreator.ts b/src/omnisharp/OmnisharpPackageCreator.ts index 87130add5..c1dc9ce8c 100644 --- a/src/omnisharp/OmnisharpPackageCreator.ts +++ b/src/omnisharp/OmnisharpPackageCreator.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package } from "../packageManager/packages"; +import { Package } from "../packageManager/Package"; export function GetPackagesFromVersion(version: string, runTimeDependencies: Package[], serverUrl: string, installPath: string): Package[] { if (!version) { diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index bb3bc4f33..3b01a3270 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -8,7 +8,7 @@ import * as util from '../common'; import * as fs from 'fs'; import { EventStream } from "../EventStream"; import { DownloadSuccess, DownloadStart, DownloadFallBack, DownloadFailure, DownloadProgress, DownloadSizeObtained } from "../omnisharp/loggingEvents"; -import { NestedError } from "./packages"; +import { NestedError } from "./Package"; import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; diff --git a/src/packageManager/packages.ts b/src/packageManager/Package.ts similarity index 100% rename from src/packageManager/packages.ts rename to src/packageManager/Package.ts diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts index 64acbd793..9bf1477b3 100644 --- a/src/packageManager/PackageFilePathResolver.ts +++ b/src/packageManager/PackageFilePathResolver.ts @@ -5,7 +5,7 @@ import * as path from 'path'; import * as util from '../common'; -import { Package } from './packages'; +import { Package } from './Package'; export function ResolveFilePaths(pkg: Package) { pkg.installTestPath = ResolvePackageTestPath(pkg); diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index f821f41b6..b480dba63 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package, PackageError } from "./packages"; +import { Package, PackageError } from "./Package"; import { PlatformInformation } from "../platform"; import * as util from '../common'; import { ResolvePackageTestPath } from "./PackageFilePathResolver"; diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 5c4b02877..7fcae53d0 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { PlatformInformation } from "../platform"; -import { Package, PackageError, NestedError } from './packages'; +import { Package, PackageError, NestedError } from './Package'; import { DownloadFile } from './FileDownloader'; import { InstallPackage } from './ZipInstaller'; import { EventStream } from '../EventStream'; diff --git a/src/packageManager/ZipInstaller.ts b/src/packageManager/ZipInstaller.ts index 6d2893d5e..7a7bc17c8 100644 --- a/src/packageManager/ZipInstaller.ts +++ b/src/packageManager/ZipInstaller.ts @@ -9,7 +9,7 @@ import * as yauzl from 'yauzl'; import * as path from 'path'; import { EventStream } from "../EventStream"; import { InstallationProgress } from "../omnisharp/loggingEvents"; -import { NestedError } from './packages'; +import { NestedError } from './Package'; export async function InstallPackage(fd: number, description: string, installPath: string, binaries: string[], eventStream: EventStream): Promise { diff --git a/src/tools/UpdatePackageDependencies.ts b/src/tools/UpdatePackageDependencies.ts index b48378f43..55ea679e2 100644 --- a/src/tools/UpdatePackageDependencies.ts +++ b/src/tools/UpdatePackageDependencies.ts @@ -5,7 +5,7 @@ import * as fs from 'fs'; import * as os from 'os'; -import { Package } from '../packageManager/packages'; +import { Package } from '../packageManager/Package'; interface PackageJSONFile diff --git a/tasks/offlinePackagingTasks.ts b/tasks/offlinePackagingTasks.ts index 3f934588e..c543d3d3b 100644 --- a/tasks/offlinePackagingTasks.ts +++ b/tasks/offlinePackagingTasks.ts @@ -20,7 +20,7 @@ import { EventStream } from '../src/EventStream'; import { getPackageJSON } from '../tasks/packageJson'; import { Logger } from '../src/logger'; import { PlatformInformation } from '../src/platform'; -import { Package } from '../src/packageManager/packages'; +import { Package } from '../src/packageManager/Package'; import { DownloadAndInstallPackages } from '../src/packageManager/PackageManager'; import NetworkSettings from '../src/NetworkSettings'; diff --git a/test/featureTests/OmnisharpPackageCreator.test.ts b/test/featureTests/OmnisharpPackageCreator.test.ts index 3eee9c99a..f4b5c1ea1 100644 --- a/test/featureTests/OmnisharpPackageCreator.test.ts +++ b/test/featureTests/OmnisharpPackageCreator.test.ts @@ -6,7 +6,7 @@ import { assert, should, expect } from "chai"; import { SetBinaryAndGetPackage, GetPackagesFromVersion } from "../../src/omnisharp/OmnisharpPackageCreator"; import { testPackageJSON } from "./testAssets/testAssets"; -import { Package } from "../../src/packageManager/packages"; +import { Package } from "../../src/packageManager/Package"; suite("GetOmnisharpPackage : Output package depends on the input package and other input parameters like serverUrl", () => { diff --git a/test/unitTests/logging/CsharpLoggerObserver.test.ts b/test/unitTests/logging/CsharpLoggerObserver.test.ts index 19b13e8c6..a4189e8ca 100644 --- a/test/unitTests/logging/CsharpLoggerObserver.test.ts +++ b/test/unitTests/logging/CsharpLoggerObserver.test.ts @@ -8,7 +8,7 @@ import { getNullChannel } from '../testAssets/Fakes'; import { CsharpLoggerObserver } from '../../../src/observers/CsharpLoggerObserver'; import { PlatformInformation } from '../../../src/platform'; import * as Event from '../../../src/omnisharp/loggingEvents'; -import { PackageError } from '../../../src/packageManager/packages'; +import { PackageError } from '../../../src/packageManager/Package'; suite("CsharpLoggerObserver", () => { suiteSetup(() => should()); diff --git a/test/unitTests/logging/TelemetryObserver.test.ts b/test/unitTests/logging/TelemetryObserver.test.ts index 022d82726..ff7743739 100644 --- a/test/unitTests/logging/TelemetryObserver.test.ts +++ b/test/unitTests/logging/TelemetryObserver.test.ts @@ -7,7 +7,7 @@ import { TelemetryObserver } from '../../../src/observers/TelemetryObserver'; import { PlatformInformation } from '../../../src/platform'; import { PackageInstallation, InstallationFailure, InstallationSuccess, TestExecutionCountReport, TelemetryEventWithMeasures, OmnisharpDelayTrackerEventMeasures, OmnisharpStart } from '../../../src/omnisharp/loggingEvents'; import { getNullTelemetryReporter } from '../testAssets/Fakes'; -import { PackageError, Package } from '../../../src/packageManager/packages'; +import { PackageError, Package } from '../../../src/packageManager/Package'; const chai = require('chai'); chai.use(require('chai-arrays')); From 24a5c03a830f15916711f331abf593fef762b87f Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 18 Apr 2018 13:50:45 -0700 Subject: [PATCH 41/64] Package installer test --- test/unitTests/Packages/ZipInstaller.test.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index 6556c6659..7b133e58a 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -56,6 +56,7 @@ suite('PackageInstaller', () => { testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); fd = await fs.open(path.resolve(testDirPath), 'r'); + util.setExtensionPath(tmpInstallDir.name); }); test('The folder is unzipped and all the files are present at the expected paths', async () => { @@ -81,15 +82,11 @@ suite('PackageInstaller', () => { for (let binaryPath of resolvedBinaryPaths) { expect(await util.fileExists(binaryPath)).to.be.true; let mode = (await fs.stat(binaryPath)).mode; - expect(mode).to.be.equal(0o755); + expect(mode & 0o7777).to.be.equal(0o755, `Expected mode for path ${binaryPath}`); } } }); - test('Error is thrown on incorrect install path', async () => { - expect(InstallPackage(fd, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; - }); - test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file expect(InstallPackage(0, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; From 9e2612489d86e90d9a1632993adb9edfb80003f3 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 18 Apr 2018 16:49:38 -0700 Subject: [PATCH 42/64] PR feedback --- src/CSharpExtDownloader.ts | 2 +- src/omnisharp/OmnisharpDownloader.ts | 2 +- src/packageManager/FileDownloader.ts | 12 ++++++------ src/packageManager/PackageFilterer.ts | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 0ef839a43..8a3611e9e 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -60,6 +60,6 @@ export class CSharpExtDownloader { return JSON.parse(JSON.stringify(this.packageJSON.runtimeDependencies)); } - return null; + throw new Error("No runtime dependencies found"); } } \ No newline at end of file diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 4cf0a3985..cd60bbe64 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -42,7 +42,7 @@ export class OmnisharpDownloader { } } - public async GetLatestVersion(serverUrl: string, latestVersionFileServerPath: string) { + public async GetLatestVersion(serverUrl: string, latestVersionFileServerPath: string): Promise { let installationStage = 'getLatestVersionInfoFile'; let description = "Latest Omnisharp Version Information"; let url = `${serverUrl}/${latestVersionFileServerPath}`; diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index 3b01a3270..46267579e 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -13,11 +13,11 @@ import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; -export async function DownloadFile(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, provider: NetworkSettingsProvider){ +export async function DownloadFile(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider){ eventStream.post(new DownloadStart(description)); try { - await downloadFile(fd, description, url, eventStream, provider); + await downloadFile(fd, description, url, eventStream, networkSettingsProvider); eventStream.post(new DownloadSuccess(` Done!`)); } catch (primaryUrlError) { @@ -27,7 +27,7 @@ export async function DownloadFile(fd: number, description: string, url: string, if (fallbackUrl) { eventStream.post(new DownloadFallBack(fallbackUrl)); try { - await downloadFile(fd, description, fallbackUrl, eventStream, provider); + await downloadFile(fd, description, fallbackUrl, eventStream, networkSettingsProvider); eventStream.post(new DownloadSuccess(' Done!')); } catch (fallbackUrlError) { @@ -40,9 +40,9 @@ export async function DownloadFile(fd: number, description: string, url: string, } } -async function downloadFile(fd: number, description: string, urlString: string, eventStream: EventStream, provider: NetworkSettingsProvider): Promise { +async function downloadFile(fd: number, description: string, urlString: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider): Promise { const url = parseUrl(urlString); - const networkSettings = provider(); + const networkSettings = networkSettingsProvider(); const proxy = networkSettings.proxy; const strictSSL = networkSettings.strictSSL; const options: https.RequestOptions = { @@ -61,7 +61,7 @@ async function downloadFile(fd: number, description: string, urlString: string, let request = https.request(options, response => { if (response.statusCode === 301 || response.statusCode === 302) { // Redirect - download from new location - return resolve(downloadFile(fd, description, response.headers.location, eventStream, provider)); + return resolve(downloadFile(fd, description, response.headers.location, eventStream, networkSettingsProvider)); } if (response.statusCode != 200) { diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index b480dba63..6e31ebbb0 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -34,7 +34,7 @@ function filterPlatformPackages(packages: Package[], platformInfo: PlatformInfor } } -async function filterAlreadyInstalledPackages(packages: Package[]) { +async function filterAlreadyInstalledPackages(packages: Package[]): Promise { return filterAsync(packages, async (pkg: Package) => { //If the file is present at the install test path then filter it return !(await util.fileExists(ResolvePackageTestPath(pkg))); From bccfa13bb2091f7c545cf43050ac7a89e0d43c37 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Wed, 18 Apr 2018 17:02:13 -0700 Subject: [PATCH 43/64] Package Filter test --- .../Packages/PackageFilterer.test.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/unitTests/Packages/PackageFilterer.test.ts diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts new file mode 100644 index 000000000..7467d722a --- /dev/null +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as chai from 'chai'; +import * as util from '../../../src/common'; +import { createTmpDir } from "../../../src/CreateTmpAsset"; +import { PlatformInformation } from "../../../src/platform"; +import { filterPackages } from "../../../src/packageManager/PackageFilterer"; +import { ResolveFilePaths } from "../../../src/packageManager/PackageFilePathResolver"; +import { Package } from "../../../src/packageManager/Package"; + +let expect = chai.expect; + +suite('PackageFilterer', () => { + const packages = [ + { + "platforms": [ "platform1" ], + "architectures": [ "architecture1" ], + "installTestPath": "path1" + }, + { + "platforms": [ "platform2" ], + "architectures": [ "architecture2" ], + "installTestPath": "path2" + }, + { + "platforms": [ "platform1" ], + "architectures": [ "architecture2" ], + "installTestPath": "path3" + }, + { + "platforms": [ "platform2" ], + "architectures": [ "architecture1" ], + "installTestPath": "path4" + }, + ]; + + setup(async () => { + let tmpDir = await createTmpDir(false); + util.setExtensionPath(tmpDir.name); + packages.forEach(pkg => ResolveFilePaths(pkg)); + }); + + test('Filters the packages based on Platform Information', async () => { + let platformInfo = new PlatformInformation("platform1", "architecture1"); + let filteredPackages = await filterPackages(packages, platformInfo); + expect(filteredPackages.length).to.be.equal(1); + expect(filteredPackages[0].platforms[0]).to.be.equal("platform1"); + expect(filteredPackages[0].architectures[0]).to.be.equal("architecture1"); + }); + + test('Returns only uninstalled packages', () => { + + }); +}); \ No newline at end of file From 2b6ba72c78eba3135b28dd9c755ddf2acf73d9e0 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 19 Apr 2018 10:52:28 -0700 Subject: [PATCH 44/64] Remove yauzl.d.ts --- package-lock.json | 10 +++++++ package.json | 3 +- src/packageManager/ZipInstaller.ts | 3 +- typings/yauzl/yauzl.d.ts | 48 ------------------------------ 4 files changed, 13 insertions(+), 51 deletions(-) delete mode 100644 typings/yauzl/yauzl.d.ts diff --git a/package-lock.json b/package-lock.json index de94b9ac3..60eeff391 100644 --- a/package-lock.json +++ b/package-lock.json @@ -194,6 +194,16 @@ "@types/node": "9.6.4" } }, + "@types/yauzl": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.0.tgz", + "integrity": "sha512-KVQbjKvieCq6d5LqZ8KIzzwygF88fWC+l7wvPbRPM3OI3f9ZAlhaKUlk3kjiyvOMqopSTM7enjduXXl5B+msXw==", + "dev": true, + "requires": { + "@types/events": "1.2.0", + "@types/node": "9.6.4" + } + }, "abbrev": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", diff --git a/package.json b/package.json index 6005cf712..234511aee 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "tmp": "0.0.33", "vscode-debugprotocol": "^1.6.1", "vscode-extension-telemetry": "0.0.15", - "yauzl": "^2.5.0" + "yauzl": "^2.9.1" }, "devDependencies": { "@types/archiver": "^2.1.1", @@ -97,6 +97,7 @@ "@types/node": "^9.4.7", "@types/semver": "^5.5.0", "@types/tmp": "0.0.33", + "@types/yauzl": "^2.9.0", "archiver": "^2.1.1", "async-child-process": "^1.1.1", "async-file": "^2.0.2", diff --git a/src/packageManager/ZipInstaller.ts b/src/packageManager/ZipInstaller.ts index 7a7bc17c8..8f8efcc70 100644 --- a/src/packageManager/ZipInstaller.ts +++ b/src/packageManager/ZipInstaller.ts @@ -5,13 +5,12 @@ import * as fs from 'fs'; import * as mkdirp from 'mkdirp'; -import * as yauzl from 'yauzl'; import * as path from 'path'; +import * as yauzl from 'yauzl'; import { EventStream } from "../EventStream"; import { InstallationProgress } from "../omnisharp/loggingEvents"; import { NestedError } from './Package'; - export async function InstallPackage(fd: number, description: string, installPath: string, binaries: string[], eventStream: EventStream): Promise { const installationStage = 'installPackages'; diff --git a/typings/yauzl/yauzl.d.ts b/typings/yauzl/yauzl.d.ts deleted file mode 100644 index 3f90f724d..000000000 --- a/typings/yauzl/yauzl.d.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -declare module 'yauzl' { - - import { EventEmitter } from 'events'; - import { Readable } from 'stream'; - - export class Entry { - fileName: string; - extraFields: { id: number; data: Buffer; }[]; - comment: string; - versionMadeBy: number; - versionNeededToExtract: number; - generalPurposeBitFlag: number; - compressionMethod: number; - lastModFileTime: number; - lastModFileDate: number; - crc32: number; - compressedSize: number; - uncompressedSize: number; - fileNameLength: number; - extraFieldLength: number; - fileCommentLength: number; - internalFileAttributes: number; - externalFileAttributes: number; - relativeOffsetOfLocalHeader: number; - getLastModDate(): Date; - } - - export class ZipFile extends EventEmitter { - readEntry(); - openReadStream(entry: Entry, callback: (err?: Error, stream?: Readable)=>void); - close(); - isOpen: boolean; - entryCount: number; - comment: string; - } - - export interface IOptions { - autoClose?: boolean; - lazyEntries?: boolean; - } - - export function fromFd(fd: number, options: IOptions, callback: (err?: Error, zipfile?: ZipFile)=>void): void; -} \ No newline at end of file From 7b2586f5f80bfd81ce79d2e62297803aa0dda8f6 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Thu, 19 Apr 2018 14:21:46 -0700 Subject: [PATCH 45/64] Filter test --- src/CreateTmpAsset.ts | 14 +++---- .../Packages/PackageFilterer.test.ts | 37 ++++++++++++++++--- test/unitTests/Packages/ZipInstaller.test.ts | 2 +- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index 3a4a5edd1..8bb7dac03 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -23,7 +23,7 @@ export async function createTmpFile(): Promise { return { fd: tmpFile.fd, name: tmpFile.name, - dispose: () => tmpFile.removeCallback() + dispose: tmpFile.removeCallback }; } @@ -37,7 +37,7 @@ export async function createTmpDir(unsafeCleanup: boolean): Promise { resolve({ name: path, removeCallback: cleanupCallback }); }); }); - + return { fd: tmpDir.fd, name: tmpDir.name, @@ -47,13 +47,13 @@ export async function createTmpDir(unsafeCleanup: boolean): Promise { } else { tmpDir.removeCallback(); - } + } } }; } export interface TmpAsset { - fd: number; - name: string; - dispose: () => void; - } \ No newline at end of file + fd: number; + name: string; + dispose: () => void; +} \ No newline at end of file diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index 7467d722a..b1b88e69e 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -4,7 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import * as chai from 'chai'; import * as util from '../../../src/common'; -import { createTmpDir } from "../../../src/CreateTmpAsset"; +import * as fs from 'async-file'; +import { createTmpDir, createTmpFile, TmpAsset } from "../../../src/CreateTmpAsset"; import { PlatformInformation } from "../../../src/platform"; import { filterPackages } from "../../../src/packageManager/PackageFilterer"; import { ResolveFilePaths } from "../../../src/packageManager/PackageFilePathResolver"; @@ -13,23 +14,36 @@ import { Package } from "../../../src/packageManager/Package"; let expect = chai.expect; suite('PackageFilterer', () => { + let tmpDir: TmpAsset; + let tmpFile: TmpAsset; const packages = [ - { + { + "description": "Platfrom1-Architecture1 uninstalled package", "platforms": [ "platform1" ], "architectures": [ "architecture1" ], "installTestPath": "path1" }, + { + //already installed package + "description": "Platfrom1-Architecture1 installed package", + "platforms": [ "platform1" ], + "architectures": [ "architecture1" ], + "installTestPath": "path5" + }, { + "description": "Platfrom2-Architecture2 uninstalled package", "platforms": [ "platform2" ], "architectures": [ "architecture2" ], "installTestPath": "path2" }, { + "description": "Platfrom1-Architecture2 uninstalled package", "platforms": [ "platform1" ], "architectures": [ "architecture2" ], "installTestPath": "path3" }, { + "description": "Platfrom2-Architecture1 uninstalled package", "platforms": [ "platform2" ], "architectures": [ "architecture1" ], "installTestPath": "path4" @@ -37,20 +51,33 @@ suite('PackageFilterer', () => { ]; setup(async () => { - let tmpDir = await createTmpDir(false); + tmpDir = await createTmpDir(true); + tmpFile = await createTmpFile(); + packages[1].installTestPath = tmpFile.name; util.setExtensionPath(tmpDir.name); packages.forEach(pkg => ResolveFilePaths(pkg)); }); test('Filters the packages based on Platform Information', async () => { + let platformInfo = new PlatformInformation("platform2", "architecture2"); + let filteredPackages = await filterPackages(packages, platformInfo); + expect(filteredPackages.length).to.be.equal(1); + expect(filteredPackages[0].description).to.be.equal("Platfrom2-Architecture2 uninstalled package"); + expect(filteredPackages[0].platforms[0]).to.be.equal("platform2"); + expect(filteredPackages[0].architectures[0]).to.be.equal("architecture2"); + }); + + test('Returns only uninstalled packages', async () => { let platformInfo = new PlatformInformation("platform1", "architecture1"); let filteredPackages = await filterPackages(packages, platformInfo); expect(filteredPackages.length).to.be.equal(1); + expect(filteredPackages[0].description).to.be.equal("Platfrom1-Architecture1 uninstalled package"); expect(filteredPackages[0].platforms[0]).to.be.equal("platform1"); expect(filteredPackages[0].architectures[0]).to.be.equal("architecture1"); }); - test('Returns only uninstalled packages', () => { - + teardown(() => { + tmpDir.dispose(); + tmpFile.dispose(); }); }); \ No newline at end of file diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index 7b133e58a..ed62981fa 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -76,7 +76,7 @@ suite('PackageInstaller', () => { }); test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { - if (!(await PlatformInformation.GetCurrent()).isWindows()) { + if (!((await PlatformInformation.GetCurrent()).isWindows())) { let resolvedBinaryPaths = binaries.map(binary => path.join(tmpInstallDir.name, binary.path)); await InstallPackage(fd, fileDescription, tmpInstallDir.name, resolvedBinaryPaths, eventStream); for (let binaryPath of resolvedBinaryPaths) { From df111f6f8b1f7ff7d3924071c409696606b68e36 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Thu, 19 Apr 2018 14:43:46 -0700 Subject: [PATCH 46/64] Added test for invalid zip file --- src/packageManager/FileDownloader.ts | 6 ++-- src/packageManager/ZipInstaller.ts | 8 +++--- .../unitTests/Packages/FileDownloader.test.ts | 2 +- .../Packages/PackageFilterer.test.ts | 7 ++--- test/unitTests/Packages/ZipInstaller.test.ts | 28 ++++++++++++------- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index 46267579e..a2537d7b1 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -13,11 +13,11 @@ import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; -export async function DownloadFile(fd: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider){ +export async function DownloadFile(destinationFileDescriptor: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider){ eventStream.post(new DownloadStart(description)); try { - await downloadFile(fd, description, url, eventStream, networkSettingsProvider); + await downloadFile(destinationFileDescriptor, description, url, eventStream, networkSettingsProvider); eventStream.post(new DownloadSuccess(` Done!`)); } catch (primaryUrlError) { @@ -27,7 +27,7 @@ export async function DownloadFile(fd: number, description: string, url: string, if (fallbackUrl) { eventStream.post(new DownloadFallBack(fallbackUrl)); try { - await downloadFile(fd, description, fallbackUrl, eventStream, networkSettingsProvider); + await downloadFile(destinationFileDescriptor, description, fallbackUrl, eventStream, networkSettingsProvider); eventStream.post(new DownloadSuccess(' Done!')); } catch (fallbackUrlError) { diff --git a/src/packageManager/ZipInstaller.ts b/src/packageManager/ZipInstaller.ts index 8f8efcc70..8f39f5ff8 100644 --- a/src/packageManager/ZipInstaller.ts +++ b/src/packageManager/ZipInstaller.ts @@ -11,17 +11,17 @@ import { EventStream } from "../EventStream"; import { InstallationProgress } from "../omnisharp/loggingEvents"; import { NestedError } from './Package'; -export async function InstallPackage(fd: number, description: string, installPath: string, binaries: string[], eventStream: EventStream): Promise { +export async function InstallPackage(sourceFileDescriptor: number, description: string, destinationInstallPath: string, binaries: string[], eventStream: EventStream): Promise { const installationStage = 'installPackages'; eventStream.post(new InstallationProgress(installationStage, description)); return new Promise((resolve, reject) => { - if (fd == 0) { + if (sourceFileDescriptor == 0) { return reject(new NestedError('Downloaded file unavailable')); } - yauzl.fromFd(fd, { lazyEntries: true }, (err, zipFile) => { + yauzl.fromFd(sourceFileDescriptor, { lazyEntries: true }, (err, zipFile) => { if (err) { return reject(new NestedError('Immediate zip file error', err)); } @@ -29,7 +29,7 @@ export async function InstallPackage(fd: number, description: string, installPat zipFile.readEntry(); zipFile.on('entry', (entry: yauzl.Entry) => { - let absoluteEntryPath = path.resolve(installPath, entry.fileName); + let absoluteEntryPath = path.resolve(destinationInstallPath, entry.fileName); if (entry.fileName.endsWith('/')) { // Directory - create it diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index b908d13a4..1816dc51f 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -17,7 +17,7 @@ chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); let expect = chai.expect; -suite("PackageDownloader", () => { +suite("FileDownloader", () => { let server = new ServerMock({ host: "localhost", port: 9000 }, { host: "localhost", diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index b1b88e69e..8b9246e1f 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -14,8 +14,8 @@ import { Package } from "../../../src/packageManager/Package"; let expect = chai.expect; suite('PackageFilterer', () => { - let tmpDir: TmpAsset; let tmpFile: TmpAsset; + const extensionPath = "ExtensionPath"; const packages = [ { "description": "Platfrom1-Architecture1 uninstalled package", @@ -51,10 +51,10 @@ suite('PackageFilterer', () => { ]; setup(async () => { - tmpDir = await createTmpDir(true); tmpFile = await createTmpFile(); packages[1].installTestPath = tmpFile.name; - util.setExtensionPath(tmpDir.name); + util.setExtensionPath(extensionPath); + // we need to set the extension path because fileresolver uses it packages.forEach(pkg => ResolveFilePaths(pkg)); }); @@ -77,7 +77,6 @@ suite('PackageFilterer', () => { }); teardown(() => { - tmpDir.dispose(); tmpFile.dispose(); }); }); \ No newline at end of file diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index ed62981fa..75c7b9245 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import * as chai from 'chai'; import * as util from '../../../src/common'; import * as archiver from 'archiver'; -import { createTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; +import { createTmpDir, TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; @@ -17,11 +17,13 @@ import { BaseEvent, InstallationProgress } from '../../../src/omnisharp/loggingE chai.use(require("chai-as-promised")); let expect = chai.expect; -suite('PackageInstaller', () => { +suite('ZipInstaller', () => { let tmpSourceDir: TmpAsset; let tmpInstallDir: TmpAsset; let testDirPath: string; - let fd: number; + let zipFileDescriptor: number; + let txtFile: TmpAsset; + let installationPath: string; const files = [ { @@ -52,23 +54,25 @@ suite('PackageInstaller', () => { eventBus = []; tmpSourceDir = await createTmpDir(true); tmpInstallDir = await createTmpDir(true); + installationPath = tmpInstallDir.name; + txtFile = await createTmpFile(); allFiles = [...files, ...binaries]; testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); - fd = await fs.open(path.resolve(testDirPath), 'r'); + zipFileDescriptor = await fs.open(path.resolve(testDirPath), 'r'); util.setExtensionPath(tmpInstallDir.name); }); test('The folder is unzipped and all the files are present at the expected paths', async () => { - await InstallPackage(fd, fileDescription, tmpInstallDir.name, [], eventStream); + await InstallPackage(zipFileDescriptor, fileDescription, installationPath, [], eventStream); for (let elem of allFiles) { - let filePath = path.join(tmpInstallDir.name, elem.path); + let filePath = path.join(installationPath, elem.path); expect(await util.fileExists(filePath)).to.be.true; } }); test('The folder is unzipped and all the expected events are created', async () => { - await InstallPackage(fd, fileDescription, tmpInstallDir.name, [], eventStream); + await InstallPackage(zipFileDescriptor, fileDescription, installationPath, [], eventStream); let eventSequence: BaseEvent[] = [ new InstallationProgress('installPackages', fileDescription) ]; @@ -77,8 +81,8 @@ suite('PackageInstaller', () => { test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { if (!((await PlatformInformation.GetCurrent()).isWindows())) { - let resolvedBinaryPaths = binaries.map(binary => path.join(tmpInstallDir.name, binary.path)); - await InstallPackage(fd, fileDescription, tmpInstallDir.name, resolvedBinaryPaths, eventStream); + let resolvedBinaryPaths = binaries.map(binary => path.join(installationPath, binary.path)); + await InstallPackage(zipFileDescriptor, fileDescription, installationPath, resolvedBinaryPaths, eventStream); for (let binaryPath of resolvedBinaryPaths) { expect(await util.fileExists(binaryPath)).to.be.true; let mode = (await fs.stat(binaryPath)).mode; @@ -87,13 +91,17 @@ suite('PackageInstaller', () => { } }); + test('Error is thrown when the file is not a zip', async () => { + expect(InstallPackage(txtFile.fd, "Text File", installationPath, [], eventStream)).to.be.rejected; + }); + test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file expect(InstallPackage(0, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; }); teardown(async () => { - await fs.close(fd); + await fs.close(zipFileDescriptor); tmpSourceDir.dispose(); tmpInstallDir.dispose(); }); From 3ff44bb29e63b4d75807d6f9af3bd5dd06cea083 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Thu, 19 Apr 2018 14:56:15 -0700 Subject: [PATCH 47/64] method for getting the request options --- .../unitTests/Packages/FileDownloader.test.ts | 45 +++++++------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 1816dc51f..7b542e0db 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -39,35 +39,10 @@ suite("FileDownloader", () => { const correctUrl = `${httpsServerUrl}${correctUrlPath}`; const redirectUrl = `${httpsServerUrl}${redirectUrlPath}`; const errorUrl = `${httpsServerUrl}${errorUrlPath}`; - - const requestOptions = { - method: 'GET', - path: correctUrlPath, - reply: { - status: 200, - headers: { "content-type": "text/plain" }, - body: "Test content" - } - }; - - const requestOptionsRedirect = { - method: 'GET', - path: redirectUrlPath, - reply: { - status: 301, - headers: { - "location": correctUrl - }, - } - }; - - const requestOptionsError = { - method: 'GET', - path: errorUrlPath, - reply: { - status: 404, - } - }; + + const requestOptions = getResponseHandlerOptions('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content"); + const requestOptionsError = getResponseHandlerOptions('GET', errorUrlPath, 404); + const requestOptionsRedirect = getResponseHandlerOptions('GET', redirectUrlPath, 301, { "location": correctUrl }); setup(async () => { await new Promise(resolve => server.start(resolve)); @@ -163,3 +138,15 @@ suite("FileDownloader", () => { } }); }); + +function getResponseHandlerOptions(method: string, path: string, reply_status: number, reply_headers?: any, reply_body?: string) { + return { + method, + path, + reply: { + status: reply_status, + headers: reply_headers, + body: reply_body + } + }; +} From ebe6a2a0f14cb51225b6cf7cac514685f1ac5123 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 19 Apr 2018 15:01:59 -0700 Subject: [PATCH 48/64] remove imports --- test/unitTests/Packages/PackageFilterer.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index 8b9246e1f..4326179df 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -4,8 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as chai from 'chai'; import * as util from '../../../src/common'; -import * as fs from 'async-file'; -import { createTmpDir, createTmpFile, TmpAsset } from "../../../src/CreateTmpAsset"; +import { createTmpFile, TmpAsset } from "../../../src/CreateTmpAsset"; import { PlatformInformation } from "../../../src/platform"; import { filterPackages } from "../../../src/packageManager/PackageFilterer"; import { ResolveFilePaths } from "../../../src/packageManager/PackageFilePathResolver"; From 3a9796816bfebafec6397a1c10d01ac93a919d01 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 20 Apr 2018 00:39:46 -0700 Subject: [PATCH 49/64] please resolve the path --- src/packageManager/PackageFilePathResolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts index 9bf1477b3..9d5d409de 100644 --- a/src/packageManager/PackageFilePathResolver.ts +++ b/src/packageManager/PackageFilePathResolver.ts @@ -33,7 +33,7 @@ function ResolvePackageBinaries(pkg: Package) { function ResolveBaseInstallPath(pkg: Package): string { let basePath = util.getExtensionPath(); if (pkg.installPath) { - basePath = path.join(basePath, pkg.installPath); + basePath = path.resolve(basePath, pkg.installPath); } return basePath; From 93ea65c62cc148aa9bbb49392ec1ed9751c1e796 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 20 Apr 2018 00:58:02 -0700 Subject: [PATCH 50/64] remove latest in settings --- .../testAssets/singleCsproj/.vscode/settings.json | 2 +- .../testAssets/slnWithCsproj/.vscode/settings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json index 15054e359..37c8593f8 100644 --- a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - "omnisharp.path": "latest" + //"omnisharp.path": "latest" } \ No newline at end of file diff --git a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json index 15054e359..37c8593f8 100644 --- a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - "omnisharp.path": "latest" + //"omnisharp.path": "latest" } \ No newline at end of file From cc4a78250de51920de1c5e5bc183017062073d74 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 20 Apr 2018 01:11:35 -0700 Subject: [PATCH 51/64] trial for travis --- .../testAssets/singleCsproj/.vscode/settings.json | 2 +- .../testAssets/slnWithCsproj/.vscode/settings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json index 37c8593f8..ce4931dd0 100644 --- a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - //"omnisharp.path": "latest" + "omnisharp.path": "1.29.1" } \ No newline at end of file diff --git a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json index 37c8593f8..ce4931dd0 100644 --- a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - //"omnisharp.path": "latest" + "omnisharp.path": "1.29.1" } \ No newline at end of file From 13ad87d3bb305e9b6dad6a352ac6acf635b61fdd Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 20 Apr 2018 13:50:40 -0700 Subject: [PATCH 52/64] Test run --- .../testAssets/singleCsproj/.vscode/settings.json | 2 +- .../testAssets/slnWithCsproj/.vscode/settings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json index ce4931dd0..320516793 100644 --- a/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/singleCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - "omnisharp.path": "1.29.1" + "omnisharp.path": "1.29.2-beta.62" } \ No newline at end of file diff --git a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json index ce4931dd0..320516793 100644 --- a/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json +++ b/test/integrationTests/testAssets/slnWithCsproj/.vscode/settings.json @@ -1,3 +1,3 @@ { - "omnisharp.path": "1.29.1" + "omnisharp.path": "1.29.2-beta.62" } \ No newline at end of file From a6ed9a05c8c5003f5adc9728a598e942f7b290b5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Apr 2018 17:05:31 -0700 Subject: [PATCH 53/64] Package Manager test executing --- src/CreateTmpAsset.ts | 4 +- src/omnisharp/OmnisharpDownloader.ts | 4 +- src/packageManager/PackageFilterer.ts | 6 ++ src/packageManager/PackageManager.ts | 4 +- .../unitTests/Packages/FileDownloader.test.ts | 5 +- .../Packages/PackageFilterer.test.ts | 4 +- .../unitTests/Packages/PackageManager.test.ts | 91 +++++++++++++++++++ test/unitTests/Packages/ZipInstaller.test.ts | 58 ++---------- test/unitTests/testAssets/CreateTestZip.ts | 52 +++++++++++ 9 files changed, 167 insertions(+), 61 deletions(-) create mode 100644 test/unitTests/Packages/PackageManager.test.ts create mode 100644 test/unitTests/testAssets/CreateTestZip.ts diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index 8bb7dac03..fb13921b2 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -6,7 +6,7 @@ import * as tmp from 'tmp'; import { NestedError } from './packageManager/Package'; import { rimraf } from 'async-file'; -export async function createTmpFile(): Promise { +export async function CreateTmpFile(): Promise { const tmpFile = await new Promise((resolve, reject) => { tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => { if (err) { @@ -27,7 +27,7 @@ export async function createTmpFile(): Promise { }; } -export async function createTmpDir(unsafeCleanup: boolean): Promise { +export async function CreateTmpDir(unsafeCleanup: boolean): Promise { const tmpDir = await new Promise((resolve, reject) => { tmp.dir({ unsafeCleanup }, (err, path, cleanupCallback) => { if (err) { diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index cd60bbe64..4678639c0 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -10,7 +10,7 @@ import { PackageInstallation, LogPlatformInfo, InstallationSuccess, Installation import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; -import { createTmpFile, TmpAsset } from '../CreateTmpAsset'; +import { CreateTmpFile, TmpAsset } from '../CreateTmpAsset'; import { DownloadFile } from '../packageManager/FileDownloader'; import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; @@ -49,7 +49,7 @@ export class OmnisharpDownloader { let tmpFile: TmpAsset; try { this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); - tmpFile = await createTmpFile(); + tmpFile = await CreateTmpFile(); await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.provider); return fs.readFileSync(tmpFile.name, 'utf8'); } diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index 6e31ebbb0..5e262f9c2 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -37,6 +37,12 @@ function filterPlatformPackages(packages: Package[], platformInfo: PlatformInfor async function filterAlreadyInstalledPackages(packages: Package[]): Promise { return filterAsync(packages, async (pkg: Package) => { //If the file is present at the install test path then filter it + let testPath = ResolvePackageTestPath(pkg); + if (!testPath) { + //if there is no testPath specified then we will not filter it + return true; + } + return !(await util.fileExists(ResolvePackageTestPath(pkg))); }); } \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 7fcae53d0..2233f75d5 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -10,7 +10,7 @@ import { InstallPackage } from './ZipInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; -import { createTmpFile, TmpAsset } from "../CreateTmpAsset"; +import { CreateTmpFile, TmpAsset } from "../CreateTmpAsset"; //Package manager needs a list of packages to be filtered based on platformInfo then download and install them //Note that the packages that this component will install needs absolute paths for the installPath, intsallTestPath and the binaries @@ -20,7 +20,7 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: if (filteredPackages) { for (let pkg of filteredPackages) { try { - tmpFile = await createTmpFile(); + tmpFile = await CreateTmpFile(); await DownloadFile(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); } diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 7b542e0db..658c8447e 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -9,7 +9,7 @@ import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; import { DownloadFile } from '../../../src/packageManager/FileDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; -import { TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; +import { TmpAsset, CreateTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; let ServerMock = require("mock-http-server"); @@ -17,6 +17,7 @@ chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); let expect = chai.expect; +//to do:look into http url thing suite("FileDownloader", () => { let server = new ServerMock({ host: "localhost", port: 9000 }, { @@ -46,7 +47,7 @@ suite("FileDownloader", () => { setup(async () => { await new Promise(resolve => server.start(resolve)); - tmpFile = await createTmpFile(); + tmpFile = await CreateTmpFile(); util.setExtensionPath(tmpFile.name); eventBus = []; server.on(requestOptions); diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index 4326179df..40a2222a8 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as chai from 'chai'; import * as util from '../../../src/common'; -import { createTmpFile, TmpAsset } from "../../../src/CreateTmpAsset"; +import { CreateTmpFile, TmpAsset } from "../../../src/CreateTmpAsset"; import { PlatformInformation } from "../../../src/platform"; import { filterPackages } from "../../../src/packageManager/PackageFilterer"; import { ResolveFilePaths } from "../../../src/packageManager/PackageFilePathResolver"; @@ -50,7 +50,7 @@ suite('PackageFilterer', () => { ]; setup(async () => { - tmpFile = await createTmpFile(); + tmpFile = await CreateTmpFile(); packages[1].installTestPath = tmpFile.name; util.setExtensionPath(extensionPath); // we need to set the extension path because fileresolver uses it diff --git a/test/unitTests/Packages/PackageManager.test.ts b/test/unitTests/Packages/PackageManager.test.ts new file mode 100644 index 000000000..e6e362109 --- /dev/null +++ b/test/unitTests/Packages/PackageManager.test.ts @@ -0,0 +1,91 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as https from 'https'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as chai from 'chai'; +import * as util from '../../../src/common'; +import { CreateTmpDir, TmpAsset } from '../../../src/CreateTmpAsset'; +import { Binaries, Files, createTestZipAsync } from '../testAssets/CreateTestZip'; +import { Package } from '../../../src/packageManager/Package'; +import { DownloadAndInstallPackages } from '../../../src/packageManager/PackageManager'; +import NetworkSettings from '../../../src/NetworkSettings'; +import { PlatformInformation } from '../../../src/platform'; +import { EventStream } from '../../../src/EventStream'; +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; + +chai.use(require("chai-as-promised")); +let expect = chai.expect; + +suite("Package Manager", () => { + let tmpSourceDir: TmpAsset; + let tmpInstallDir: TmpAsset; + let server: https.Server; + let testDirPath: string; + let allFiles: Array<{ content: string, path: string }>; + let installationPath: string; + let eventBus: Array; + let packages: Package[]; + + const packageDescription = "Test Package"; + const eventStream = new EventStream(); + eventStream.subscribe(event => eventBus.push(event)); + + const platformInfo = new PlatformInformation("win32", "x86"); + const networkSettingsProvider = () => new NetworkSettings(undefined, false); + const options = { + key: fs.readFileSync("test/unitTests/testAssets/private.pem"), + cert: fs.readFileSync("test/unitTests/testAssets/public.pem") + }; + + setup(async () => { + eventBus = []; + tmpSourceDir = await CreateTmpDir(true); + tmpInstallDir = await CreateTmpDir(true); + installationPath = tmpInstallDir.name; + packages = [{ url: "https://localhost:8000", description: packageDescription, installPath: installationPath }]; + allFiles = [...Files, ...Binaries]; + testDirPath = tmpSourceDir.name + "/test.zip"; + await createTestZipAsync(testDirPath, allFiles); + server = https.createServer(options, (req, response) => { + let stat = fs.statSync(testDirPath); + response.writeHead(200, { + 'Content-Type': 'application/zip', + 'Content-Length': stat.size + }); + + let readStream = fs.createReadStream(testDirPath); + readStream.pipe(response); + }).listen(8000); + }); + + test("Downloads the package and installs at the specified path", async () => { + await DownloadAndInstallPackages(packages, networkSettingsProvider, platformInfo, eventStream); + for (let elem of allFiles) { + let filePath = path.join(installationPath, elem.path); + expect(await util.fileExists(filePath)).to.be.true; + } + }); + + test("Events are created in the correct order", async () => { + let eventsSequence = [ + new DownloadStart(packageDescription), + new DownloadSizeObtained(396), + new DownloadProgress(100, packageDescription), + new DownloadSuccess(' Done!'), + new InstallationProgress('installPackages', packageDescription) + ]; + + await DownloadAndInstallPackages(packages, networkSettingsProvider, platformInfo, eventStream); + expect(eventBus).to.be.deep.equal(eventsSequence); + }); + + teardown(() => { + tmpSourceDir.dispose(); + tmpInstallDir.dispose(); + server.close(); + }); +}); \ No newline at end of file diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index 75c7b9245..bcd2a4086 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -7,12 +7,12 @@ import * as fs from 'async-file'; import * as path from 'path'; import * as chai from 'chai'; import * as util from '../../../src/common'; -import * as archiver from 'archiver'; -import { createTmpDir, TmpAsset, createTmpFile } from '../../../src/CreateTmpAsset'; +import { CreateTmpDir, TmpAsset, CreateTmpFile } from '../../../src/CreateTmpAsset'; import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; import { BaseEvent, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; +import { Files, Binaries, createTestZipAsync } from '../testAssets/CreateTestZip'; chai.use(require("chai-as-promised")); let expect = chai.expect; @@ -25,25 +25,6 @@ suite('ZipInstaller', () => { let txtFile: TmpAsset; let installationPath: string; - const files = [ - { - content: "File 1", - path: "file1.txt", - - }, - { - content: "File 2", - path: "folder/file2.txt" - } - ]; - - const binaries = [ - { - content: "Binary 1", - path: "binary1.txt" - }, - ]; - const fileDescription = "somefile"; const eventStream = new EventStream(); let eventBus: BaseEvent[]; @@ -52,11 +33,11 @@ suite('ZipInstaller', () => { setup(async () => { eventBus = []; - tmpSourceDir = await createTmpDir(true); - tmpInstallDir = await createTmpDir(true); + tmpSourceDir = await CreateTmpDir(true); + tmpInstallDir = await CreateTmpDir(true); installationPath = tmpInstallDir.name; - txtFile = await createTmpFile(); - allFiles = [...files, ...binaries]; + txtFile = await CreateTmpFile(); + allFiles = [...Files, ...Binaries]; testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); zipFileDescriptor = await fs.open(path.resolve(testDirPath), 'r'); @@ -81,7 +62,7 @@ suite('ZipInstaller', () => { test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { if (!((await PlatformInformation.GetCurrent()).isWindows())) { - let resolvedBinaryPaths = binaries.map(binary => path.join(installationPath, binary.path)); + let resolvedBinaryPaths = Binaries.map(binary => path.join(installationPath, binary.path)); await InstallPackage(zipFileDescriptor, fileDescription, installationPath, resolvedBinaryPaths, eventStream); for (let binaryPath of resolvedBinaryPaths) { expect(await util.fileExists(binaryPath)).to.be.true; @@ -105,29 +86,4 @@ suite('ZipInstaller', () => { tmpSourceDir.dispose(); tmpInstallDir.dispose(); }); - - async function createTestZipAsync(dirPath: string, filesToAdd: Array<{ content: string, path: string }>): Promise<{}> { - let output = fs.createWriteStream(dirPath); - - return new Promise((resolve, reject) => { - output.on('close', function () { - resolve(); // the installer needs to wait for the filestream to be closed here - }); - - let archive = archiver('zip'); - archive.on('warning', function (err: any) { - if (err.code === 'ENOENT') { - console.log(err); - } else { - // throw error - reject(err); - } - }); - - archive.on('error', reject); - archive.pipe(output); - filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); - archive.finalize(); - }); - } }); diff --git a/test/unitTests/testAssets/CreateTestZip.ts b/test/unitTests/testAssets/CreateTestZip.ts new file mode 100644 index 000000000..8b0ce5ee3 --- /dev/null +++ b/test/unitTests/testAssets/CreateTestZip.ts @@ -0,0 +1,52 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as archiver from 'archiver'; +import * as fs from 'async-file'; + +export const Files = [ + { + content: "File 1", + path: "file1.txt", + + }, + { + content: "File 2", + path: "folder/file2.txt" + } +]; + +export const Binaries = [ + { + content: "Binary 1", + path: "binary1.txt" + }, +]; + + +export async function createTestZipAsync(dirPath: string, filesToAdd: Array<{ content: string, path: string }>): Promise<{}> { + let output = fs.createWriteStream(dirPath); + + return new Promise((resolve, reject) => { + output.on('close', function () { + resolve(); // the installer needs to wait for the filestream to be closed here + }); + + let archive = archiver('zip'); + archive.on('warning', function (err: any) { + if (err.code === 'ENOENT') { + console.log(err); + } else { + // throw error + reject(err); + } + }); + + archive.on('error', reject); + archive.pipe(output); + filesToAdd.forEach(elem => archive.append(elem.content, { name: elem.path })); + archive.finalize(); + }); +} \ No newline at end of file From fdc9e0c002807b0c3c3b8b660cd6dd332593c4e9 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 20 Apr 2018 17:32:49 -0700 Subject: [PATCH 54/64] Use free port in package manager test --- package-lock.json | 6 ++++++ package.json | 1 + test/unitTests/Packages/FileDownloader.test.ts | 2 +- test/unitTests/Packages/PackageManager.test.ts | 6 ++++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 60eeff391..0f665b37a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2520,6 +2520,12 @@ "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", "dev": true }, + "get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=", + "dev": true + }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", diff --git a/package.json b/package.json index 234511aee..33377662d 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "copyfiles": "^2.0.0", "cross-env": "^5.1.4", "del": "3.0.0", + "get-port": "^3.2.0", "glob-promise": "^3.4.0", "gulp": "3.9.1", "gulp-mocha": "^5.0.0", diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 658c8447e..9933e1375 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -40,7 +40,7 @@ suite("FileDownloader", () => { const correctUrl = `${httpsServerUrl}${correctUrlPath}`; const redirectUrl = `${httpsServerUrl}${redirectUrlPath}`; const errorUrl = `${httpsServerUrl}${errorUrlPath}`; - + const requestOptions = getResponseHandlerOptions('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content"); const requestOptionsError = getResponseHandlerOptions('GET', errorUrlPath, 404); const requestOptionsRedirect = getResponseHandlerOptions('GET', redirectUrlPath, 301, { "location": correctUrl }); diff --git a/test/unitTests/Packages/PackageManager.test.ts b/test/unitTests/Packages/PackageManager.test.ts index e6e362109..4aea65c95 100644 --- a/test/unitTests/Packages/PackageManager.test.ts +++ b/test/unitTests/Packages/PackageManager.test.ts @@ -16,6 +16,7 @@ import NetworkSettings from '../../../src/NetworkSettings'; import { PlatformInformation } from '../../../src/platform'; import { EventStream } from '../../../src/EventStream'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; +const getPort = require('get-port'); chai.use(require("chai-as-promised")); let expect = chai.expect; @@ -46,10 +47,11 @@ suite("Package Manager", () => { tmpSourceDir = await CreateTmpDir(true); tmpInstallDir = await CreateTmpDir(true); installationPath = tmpInstallDir.name; - packages = [{ url: "https://localhost:8000", description: packageDescription, installPath: installationPath }]; allFiles = [...Files, ...Binaries]; testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); + let port = await getPort(); + packages = [{ url: `https://localhost:${port}`, description: packageDescription, installPath: installationPath }]; server = https.createServer(options, (req, response) => { let stat = fs.statSync(testDirPath); response.writeHead(200, { @@ -59,7 +61,7 @@ suite("Package Manager", () => { let readStream = fs.createReadStream(testDirPath); readStream.pipe(response); - }).listen(8000); + }).listen(port); }); test("Downloads the package and installs at the specified path", async () => { From 16e1c885f7624c8adf4f986875ca355388b78e2f Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 11:47:39 -0700 Subject: [PATCH 55/64] Package Manager (using a https server running) --- .../unitTests/Packages/FileDownloader.test.ts | 87 +++++++++---------- .../Packages/PackageFilterer.test.ts | 20 +++++ .../MockHttpServerRequestOptions.ts | 27 ++++++ 3 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 test/unitTests/testAssets/MockHttpServerRequestOptions.ts diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 9933e1375..c42fa49cd 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -11,39 +11,48 @@ import { DownloadFile } from '../../../src/packageManager/FileDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpAsset, CreateTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; +import { getResponseHandlerOptions, MockHttpServerRequestOptions } from '../testAssets/MockHttpServerRequestOptions'; -let ServerMock = require("mock-http-server"); +const getPort = require('get-port'); +const ServerMock = require("mock-http-server"); chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); -let expect = chai.expect; +const expect = chai.expect; //to do:look into http url thing suite("FileDownloader", () => { - let server = new ServerMock({ host: "localhost", port: 9000 }, - { - host: "localhost", - port: 8080, - key: fs.readFileSync("test/unitTests/testAssets/private.pem"), - cert: fs.readFileSync("test/unitTests/testAssets/public.pem") - }); - - let tmpFile: TmpAsset; - const eventStream = new EventStream(); const fileDescription = "Test file"; - const networkSettingsProvider = () => new NetworkSettings(undefined, false); - let eventBus: BaseEvent[]; - eventStream.subscribe((event) => eventBus.push(event)); - const httpsServerUrl = "https://127.0.0.1:8080"; const correctUrlPath = `/resource`; const redirectUrlPath = '/redirectResource'; const errorUrlPath = '/errorResource'; - const correctUrl = `${httpsServerUrl}${correctUrlPath}`; - const redirectUrl = `${httpsServerUrl}${redirectUrlPath}`; - const errorUrl = `${httpsServerUrl}${errorUrlPath}`; + const networkSettingsProvider = () => new NetworkSettings(undefined, false); + const eventStream = new EventStream(); + let eventBus: BaseEvent[]; + eventStream.subscribe((event) => eventBus.push(event)); + + let server: any; + let httpsServerUrl: string; + let tmpFile: TmpAsset; + let requestOptions: MockHttpServerRequestOptions; + let requestOptionsError: MockHttpServerRequestOptions; + let requestOptionsRedirect: MockHttpServerRequestOptions; + + suiteSetup(async () => { + let port = await getPort(); + server = new ServerMock(null, + { + host: "localhost", + port: port, + key: fs.readFileSync("test/unitTests/testAssets/private.pem"), + cert: fs.readFileSync("test/unitTests/testAssets/public.pem") + }); + + httpsServerUrl = `https://127.0.0.1:${port}`; + requestOptions = getResponseHandlerOptions('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content"); + requestOptionsError = getResponseHandlerOptions('GET', errorUrlPath, 404); + requestOptionsRedirect = getResponseHandlerOptions('GET', redirectUrlPath, 301, { "location": `${httpsServerUrl}${correctUrlPath}` }); + }); - const requestOptions = getResponseHandlerOptions('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content"); - const requestOptionsError = getResponseHandlerOptions('GET', errorUrlPath, 404); - const requestOptionsRedirect = getResponseHandlerOptions('GET', redirectUrlPath, 301, { "location": correctUrl }); setup(async () => { await new Promise(resolve => server.start(resolve)); @@ -60,8 +69,8 @@ suite("FileDownloader", () => { [ { description: "Primary url", - url: correctUrl, - fallBackUrl: "", + urlPath: correctUrlPath, + fallBackUrlPath: "", eventsSequence: [ new DownloadStart(fileDescription), new DownloadSizeObtained(12), @@ -70,12 +79,12 @@ suite("FileDownloader", () => { }, { description: "Fallback url", - url: errorUrl, - fallBackUrl: correctUrl, + urlPath: errorUrlPath, + fallBackUrlPath: correctUrlPath, eventsSequence: [ new DownloadStart(fileDescription), new DownloadFailure("failed (error code '404')"), - new DownloadFallBack(correctUrl), + new DownloadFallBack(`${httpsServerUrl}${correctUrlPath}`), new DownloadSizeObtained(12), new DownloadProgress(100, fileDescription), new DownloadSuccess(' Done!')] @@ -83,7 +92,7 @@ suite("FileDownloader", () => { ].forEach((elem) => { suite(elem.description, () => { test('File is downloaded', async () => { - await DownloadFile(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${elem.urlPath}`, `${httpsServerUrl}${elem.fallBackUrlPath}`, eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); @@ -91,7 +100,7 @@ suite("FileDownloader", () => { }); test('Events are created in the correct order', async () => { - await DownloadFile(tmpFile.fd, fileDescription, elem.url, elem.fallBackUrl, eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${elem.urlPath}`, `${httpsServerUrl}${elem.fallBackUrlPath}`, eventStream, networkSettingsProvider); expect(eventBus).to.be.deep.equal(elem.eventsSequence); }); }); @@ -100,7 +109,7 @@ suite("FileDownloader", () => { suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { - await DownloadFile(tmpFile.fd, fileDescription, redirectUrl, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${redirectUrlPath}`, "", eventStream, networkSettingsProvider); const stats = fs.statSync(tmpFile.name); expect(stats.size).to.not.equal(0); let text = fs.readFileSync(tmpFile.name, "utf8"); @@ -110,7 +119,7 @@ suite("FileDownloader", () => { suite('If the response status code is not 301, 302 or 200 then the download fails', () => { test('Error is thrown', async () => { - expect(DownloadFile(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); + expect(DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); }); test('Download Start and Download Failure events are created', async () => { @@ -119,7 +128,7 @@ suite("FileDownloader", () => { new DownloadFailure("failed (error code '404')") ]; try { - await DownloadFile(tmpFile.fd, fileDescription, errorUrl, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider); } catch (error) { expect(eventBus).to.be.deep.equal(eventsSequence); @@ -129,7 +138,7 @@ suite("FileDownloader", () => { test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file - expect(DownloadFile(0, fileDescription, errorUrl, "", eventStream, networkSettingsProvider)).to.be.rejected; + expect(DownloadFile(0, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider)).to.be.rejected; }); teardown(async () => { @@ -140,14 +149,4 @@ suite("FileDownloader", () => { }); }); -function getResponseHandlerOptions(method: string, path: string, reply_status: number, reply_headers?: any, reply_body?: string) { - return { - method, - path, - reply: { - status: reply_status, - headers: reply_headers, - body: reply_body - } - }; -} + diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index 40a2222a8..f18c84e7a 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -47,6 +47,17 @@ suite('PackageFilterer', () => { "architectures": [ "architecture1" ], "installTestPath": "path4" }, + { + "description": "Platfrom1-Architecture2 uninstalled package", + "platforms": [ "platform1" ], + "architectures": [ "architecture2" ], + "installTestPath": "path3" + }, + { + "description": "Platfrom3-Architecture3 with no installTestPath specified", + "platforms": [ "platform3" ], + "architectures": [ "architecture3" ], + }, ]; setup(async () => { @@ -75,6 +86,15 @@ suite('PackageFilterer', () => { expect(filteredPackages[0].architectures[0]).to.be.equal("architecture1"); }); + test('Doesnot filter the package if install test path is not specified', async () => { + let platformInfo = new PlatformInformation("platform3", "architecture3"); + let filteredPackages = await filterPackages(packages, platformInfo); + expect(filteredPackages.length).to.be.equal(1); + expect(filteredPackages[0].description).to.be.equal("Platfrom3-Architecture3 with no installTestPath specified"); + expect(filteredPackages[0].platforms[0]).to.be.equal("platform3"); + expect(filteredPackages[0].architectures[0]).to.be.equal("architecture3"); + }); + teardown(() => { tmpFile.dispose(); }); diff --git a/test/unitTests/testAssets/MockHttpServerRequestOptions.ts b/test/unitTests/testAssets/MockHttpServerRequestOptions.ts new file mode 100644 index 000000000..177b3dd0d --- /dev/null +++ b/test/unitTests/testAssets/MockHttpServerRequestOptions.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export interface MockHttpServerRequestOptions { + method: string; + path: string; + reply: { + status: number; + headers?: any; + body: any; + }; + +} + +export function getResponseHandlerOptions(method: string, path: string, reply_status: number, reply_headers?: any, reply_body?: any): MockHttpServerRequestOptions { + return { + method, + path, + reply: { + status: reply_status, + headers: reply_headers, + body: reply_body + } + }; +} \ No newline at end of file From 1fe14a5d776940dc3205af9d02bf7ba5157e9f9b Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 14:40:46 -0700 Subject: [PATCH 56/64] using http mock server --- src/observers/CsharpLoggerObserver.ts | 9 +- src/observers/OmnisharpStatusBarObserver.ts | 6 +- src/omnisharp/OmnisharpDownloader.ts | 4 +- src/omnisharp/loggingEvents.ts | 7 +- src/packageManager/PackageManager.ts | 4 +- src/packageManager/ZipInstaller.ts | 8 +- .../unitTests/Packages/FileDownloader.test.ts | 30 +++---- .../unitTests/Packages/PackageManager.test.ts | 88 ++++++++++++------- test/unitTests/Packages/ZipInstaller.test.ts | 16 ++-- .../logging/CsharpLoggerObserver.test.ts | 2 +- .../OmnisharpStatusBarObserver.test.ts | 4 +- ...ons.ts => MockHttpServerRequestHandler.ts} | 4 +- 12 files changed, 103 insertions(+), 79 deletions(-) rename test/unitTests/testAssets/{MockHttpServerRequestOptions.ts => MockHttpServerRequestHandler.ts} (75%) diff --git a/src/observers/CsharpLoggerObserver.ts b/src/observers/CsharpLoggerObserver.ts index 139ed6a84..b67044868 100644 --- a/src/observers/CsharpLoggerObserver.ts +++ b/src/observers/CsharpLoggerObserver.ts @@ -27,8 +27,8 @@ export class CsharpLoggerObserver extends BaseLoggerObserver { this.logger.appendLine('Finished'); this.logger.appendLine(); break; - case Event.InstallationProgress.name: - this.handleInstallationProgress(event); + case Event.InstallationStart.name: + this.handleInstallationStart(event); break; case Event.DownloadStart.name: this.handleDownloadStart(event); @@ -51,6 +51,9 @@ export class CsharpLoggerObserver extends BaseLoggerObserver { case Event.DownloadSizeObtained.name: this.handleDownloadSizeObtained(event); break; + case Event.LatestBuildDownloadStart.name: + this.logger.appendLine("Getting latest OmniSharp version information"); + break; } } @@ -104,7 +107,7 @@ export class CsharpLoggerObserver extends BaseLoggerObserver { this.dots = 0; } - private handleInstallationProgress(event: Event.InstallationProgress) { + private handleInstallationStart(event: Event.InstallationStart) { this.logger.appendLine(`Installing package '${event.packageDescription}'`); this.logger.appendLine(); } diff --git a/src/observers/OmnisharpStatusBarObserver.ts b/src/observers/OmnisharpStatusBarObserver.ts index 2d5cab661..e25fe6812 100644 --- a/src/observers/OmnisharpStatusBarObserver.ts +++ b/src/observers/OmnisharpStatusBarObserver.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { OmnisharpServerOnServerError, BaseEvent, OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnStop, OmnisharpServerOnStart, DownloadStart, InstallationProgress, DownloadProgress } from "../omnisharp/loggingEvents"; +import { OmnisharpServerOnServerError, BaseEvent, OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnStop, OmnisharpServerOnStart, DownloadStart, InstallationStart, DownloadProgress } from "../omnisharp/loggingEvents"; import { BaseStatusBarItemObserver } from './BaseStatusBarItemObserver'; export class OmnisharpStatusBarObserver extends BaseStatusBarItemObserver { @@ -27,8 +27,8 @@ export class OmnisharpStatusBarObserver extends BaseStatusBarItemObserver { case DownloadStart.name: this.SetAndShowStatusBar("$(cloud-download) Downloading packages", '', '', `Downloading package '${(event).packageDescription}...' `); break; - case InstallationProgress.name: - this.SetAndShowStatusBar("$(desktop-download) Installing packages...", '', '', `Installing package '${(event).packageDescription}'`); + case InstallationStart.name: + this.SetAndShowStatusBar("$(desktop-download) Installing packages...", '', '', `Installing package '${(event).packageDescription}'`); break; case DownloadProgress.name: let progressEvent = event; diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 4678639c0..6ea1be97a 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -6,7 +6,7 @@ import * as fs from 'fs'; import { GetPackagesFromVersion } from './OmnisharpPackageCreator'; import { PlatformInformation } from '../platform'; -import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents'; +import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, LatestBuildDownloadStart } from './loggingEvents'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from '../NetworkSettings'; import { DownloadAndInstallPackages } from '../packageManager/PackageManager'; @@ -48,7 +48,7 @@ export class OmnisharpDownloader { let url = `${serverUrl}/${latestVersionFileServerPath}`; let tmpFile: TmpAsset; try { - this.eventStream.post(new InstallationProgress(installationStage, 'Getting latest build information...')); + this.eventStream.post(new LatestBuildDownloadStart()); tmpFile = await CreateTmpFile(); await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.provider); return fs.readFileSync(tmpFile.name, 'utf8'); diff --git a/src/omnisharp/loggingEvents.ts b/src/omnisharp/loggingEvents.ts index 13e9683d3..5eda58ff4 100644 --- a/src/omnisharp/loggingEvents.ts +++ b/src/omnisharp/loggingEvents.ts @@ -38,8 +38,8 @@ export class LogPlatformInfo implements BaseEvent { constructor(public info: PlatformInformation) { } } -export class InstallationProgress implements BaseEvent { - constructor(public stage: string, public packageDescription: string) { } +export class InstallationStart implements BaseEvent { + constructor(public packageDescription: string) { } } export class InstallationFailure implements BaseEvent { @@ -141,4 +141,5 @@ export class OmnisharpOnBeforeServerStart implements BaseEvent { } export class OmnisharpOnBeforeServerInstall implements BaseEvent { } export class ActiveTextEditorChanged implements BaseEvent { } export class OmnisharpServerOnStop implements BaseEvent { } -export class OmnisharpServerOnStart implements BaseEvent { } \ No newline at end of file +export class OmnisharpServerOnStart implements BaseEvent { } +export class LatestBuildDownloadStart implements BaseEvent { } \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 2233f75d5..ed0a72696 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -6,7 +6,7 @@ import { PlatformInformation } from "../platform"; import { Package, PackageError, NestedError } from './Package'; import { DownloadFile } from './FileDownloader'; -import { InstallPackage } from './ZipInstaller'; +import { InstallZip } from './ZipInstaller'; import { EventStream } from '../EventStream'; import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; @@ -22,7 +22,7 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: try { tmpFile = await CreateTmpFile(); await DownloadFile(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); - await InstallPackage(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); + await InstallZip(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); } catch (error) { if (error instanceof NestedError) { diff --git a/src/packageManager/ZipInstaller.ts b/src/packageManager/ZipInstaller.ts index 8f39f5ff8..b31311e21 100644 --- a/src/packageManager/ZipInstaller.ts +++ b/src/packageManager/ZipInstaller.ts @@ -8,13 +8,11 @@ import * as mkdirp from 'mkdirp'; import * as path from 'path'; import * as yauzl from 'yauzl'; import { EventStream } from "../EventStream"; -import { InstallationProgress } from "../omnisharp/loggingEvents"; +import { InstallationStart } from "../omnisharp/loggingEvents"; import { NestedError } from './Package'; -export async function InstallPackage(sourceFileDescriptor: number, description: string, destinationInstallPath: string, binaries: string[], eventStream: EventStream): Promise { - const installationStage = 'installPackages'; - - eventStream.post(new InstallationProgress(installationStage, description)); +export async function InstallZip(sourceFileDescriptor: number, description: string, destinationInstallPath: string, binaries: string[], eventStream: EventStream): Promise { + eventStream.post(new InstallationStart(description)); return new Promise((resolve, reject) => { if (sourceFileDescriptor == 0) { diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index c42fa49cd..5a38f99fb 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as fs from 'fs'; +import * as fs from 'async-file'; import * as chai from 'chai'; import * as util from '../../../src/common'; import { EventStream } from '../../../src/EventStream'; @@ -11,7 +11,7 @@ import { DownloadFile } from '../../../src/packageManager/FileDownloader'; import NetworkSettings from '../../../src/NetworkSettings'; import { TmpAsset, CreateTmpFile } from '../../../src/CreateTmpAsset'; import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, DownloadFallBack, DownloadFailure } from '../../../src/omnisharp/loggingEvents'; -import { getResponseHandlerOptions, MockHttpServerRequestOptions } from '../testAssets/MockHttpServerRequestOptions'; +import { getRequestHandler } from '../testAssets/MockHttpServerRequestHandler'; const getPort = require('get-port'); const ServerMock = require("mock-http-server"); @@ -33,24 +33,18 @@ suite("FileDownloader", () => { let server: any; let httpsServerUrl: string; let tmpFile: TmpAsset; - let requestOptions: MockHttpServerRequestOptions; - let requestOptionsError: MockHttpServerRequestOptions; - let requestOptionsRedirect: MockHttpServerRequestOptions; - + suiteSetup(async () => { let port = await getPort(); server = new ServerMock(null, { host: "localhost", port: port, - key: fs.readFileSync("test/unitTests/testAssets/private.pem"), - cert: fs.readFileSync("test/unitTests/testAssets/public.pem") + key: await fs.readFile("test/unitTests/testAssets/private.pem"), + cert: await fs.readFile("test/unitTests/testAssets/public.pem") }); httpsServerUrl = `https://127.0.0.1:${port}`; - requestOptions = getResponseHandlerOptions('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content"); - requestOptionsError = getResponseHandlerOptions('GET', errorUrlPath, 404); - requestOptionsRedirect = getResponseHandlerOptions('GET', redirectUrlPath, 301, { "location": `${httpsServerUrl}${correctUrlPath}` }); }); @@ -59,9 +53,9 @@ suite("FileDownloader", () => { tmpFile = await CreateTmpFile(); util.setExtensionPath(tmpFile.name); eventBus = []; - server.on(requestOptions); - server.on(requestOptionsError); - server.on(requestOptionsRedirect); + server.on(getRequestHandler('GET', correctUrlPath, 200, { "content-type": "text/plain" }, "Test content")); + server.on(getRequestHandler('GET', errorUrlPath, 404)); + server.on(getRequestHandler('GET', redirectUrlPath, 301, { "location": `${httpsServerUrl}${correctUrlPath}` })); }); suite('If the response status Code is 200, the download succeeds', () => { @@ -93,9 +87,9 @@ suite("FileDownloader", () => { suite(elem.description, () => { test('File is downloaded', async () => { await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${elem.urlPath}`, `${httpsServerUrl}${elem.fallBackUrlPath}`, eventStream, networkSettingsProvider); - const stats = fs.statSync(tmpFile.name); + const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); - let text = fs.readFileSync(tmpFile.name, "utf8"); + let text = await fs.readFile(tmpFile.name, 'utf8'); expect(text).to.be.equal("Test content"); }); @@ -110,9 +104,9 @@ suite("FileDownloader", () => { suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${redirectUrlPath}`, "", eventStream, networkSettingsProvider); - const stats = fs.statSync(tmpFile.name); + const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); - let text = fs.readFileSync(tmpFile.name, "utf8"); + let text = await fs.readFile(tmpFile.name, "utf8"); expect(text).to.be.equal("Test content"); }); }); diff --git a/test/unitTests/Packages/PackageManager.test.ts b/test/unitTests/Packages/PackageManager.test.ts index 4aea65c95..ae00b1bd0 100644 --- a/test/unitTests/Packages/PackageManager.test.ts +++ b/test/unitTests/Packages/PackageManager.test.ts @@ -3,8 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as https from 'https'; -import * as fs from 'fs'; +import * as fs from 'async-file'; import * as path from 'path'; import * as chai from 'chai'; import * as util from '../../../src/common'; @@ -15,16 +14,19 @@ import { DownloadAndInstallPackages } from '../../../src/packageManager/PackageM import NetworkSettings from '../../../src/NetworkSettings'; import { PlatformInformation } from '../../../src/platform'; import { EventStream } from '../../../src/EventStream'; -import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; -const getPort = require('get-port'); +import { BaseEvent, DownloadStart, DownloadSizeObtained, DownloadProgress, DownloadSuccess, InstallationStart } from '../../../src/omnisharp/loggingEvents'; +import { getRequestHandler } from '../testAssets/MockHttpServerRequestHandler'; chai.use(require("chai-as-promised")); -let expect = chai.expect; +const expect = chai.expect; +const ServerMock = require("mock-http-server"); +const getPort = require('get-port'); suite("Package Manager", () => { let tmpSourceDir: TmpAsset; let tmpInstallDir: TmpAsset; - let server: https.Server; + let server: any; + let downloadUrl: string; let testDirPath: string; let allFiles: Array<{ content: string, path: string }>; let installationPath: string; @@ -35,37 +37,51 @@ suite("Package Manager", () => { const eventStream = new EventStream(); eventStream.subscribe(event => eventBus.push(event)); - const platformInfo = new PlatformInformation("win32", "x86"); + const windowsPlatformInfo = new PlatformInformation("win32", "x86"); + const linuxPlatformInfo = new PlatformInformation("linux", "x86"); const networkSettingsProvider = () => new NetworkSettings(undefined, false); - const options = { - key: fs.readFileSync("test/unitTests/testAssets/private.pem"), - cert: fs.readFileSync("test/unitTests/testAssets/public.pem") - }; + + suiteSetup(async () => { + let port = await getPort(); + downloadUrl = `https://localhost:${port}/package`; + server = new ServerMock(null, + { + host: "localhost", + port: port, + key: await fs.readFile("test/unitTests/testAssets/private.pem"), + cert: await fs.readFile("test/unitTests/testAssets/public.pem") + }); + + }); setup(async () => { eventBus = []; tmpSourceDir = await CreateTmpDir(true); tmpInstallDir = await CreateTmpDir(true); installationPath = tmpInstallDir.name; + packages = [ + { + url: downloadUrl, + description: packageDescription, + installPath: installationPath, + platforms: [windowsPlatformInfo.platform], + architectures: [windowsPlatformInfo.architecture] + }]; allFiles = [...Files, ...Binaries]; testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); - let port = await getPort(); - packages = [{ url: `https://localhost:${port}`, description: packageDescription, installPath: installationPath }]; - server = https.createServer(options, (req, response) => { - let stat = fs.statSync(testDirPath); - response.writeHead(200, { - 'Content-Type': 'application/zip', - 'Content-Length': stat.size - }); - - let readStream = fs.createReadStream(testDirPath); - readStream.pipe(response); - }).listen(port); + await new Promise(resolve => server.start(resolve)); //start the server + let buffer: any; + let stat = await fs.stat(testDirPath); + buffer = await fs.readFile(testDirPath); + server.on(getRequestHandler('GET', '/package', 200, { + "content-type": "application/zip", + "content-length": stat.size + }, buffer)); }); test("Downloads the package and installs at the specified path", async () => { - await DownloadAndInstallPackages(packages, networkSettingsProvider, platformInfo, eventStream); + await DownloadAndInstallPackages(packages, networkSettingsProvider, windowsPlatformInfo, eventStream); for (let elem of allFiles) { let filePath = path.join(installationPath, elem.path); expect(await util.fileExists(filePath)).to.be.true; @@ -78,16 +94,28 @@ suite("Package Manager", () => { new DownloadSizeObtained(396), new DownloadProgress(100, packageDescription), new DownloadSuccess(' Done!'), - new InstallationProgress('installPackages', packageDescription) + new InstallationStart(packageDescription) ]; - await DownloadAndInstallPackages(packages, networkSettingsProvider, platformInfo, eventStream); + await DownloadAndInstallPackages(packages, networkSettingsProvider, windowsPlatformInfo, eventStream); expect(eventBus).to.be.deep.equal(eventsSequence); }); - teardown(() => { - tmpSourceDir.dispose(); - tmpInstallDir.dispose(); - server.close(); + test("Installs only the platform specific packages", async () => { + //since there is no linux package specified no package should be installed + await DownloadAndInstallPackages(packages, networkSettingsProvider, linuxPlatformInfo, eventStream); + let files = await fs.readdir(tmpInstallDir.name); + expect(files.length).to.equal(0); + }); + + teardown(async () => { + if (tmpSourceDir) { + tmpSourceDir.dispose(); + } + if (tmpInstallDir) { + tmpInstallDir.dispose(); + } + + await new Promise((resolve, reject) => server.stop(resolve)); }); }); \ No newline at end of file diff --git a/test/unitTests/Packages/ZipInstaller.test.ts b/test/unitTests/Packages/ZipInstaller.test.ts index bcd2a4086..930b1ec14 100644 --- a/test/unitTests/Packages/ZipInstaller.test.ts +++ b/test/unitTests/Packages/ZipInstaller.test.ts @@ -8,10 +8,10 @@ import * as path from 'path'; import * as chai from 'chai'; import * as util from '../../../src/common'; import { CreateTmpDir, TmpAsset, CreateTmpFile } from '../../../src/CreateTmpAsset'; -import { InstallPackage } from '../../../src/packageManager/ZipInstaller'; +import { InstallZip } from '../../../src/packageManager/ZipInstaller'; import { EventStream } from '../../../src/EventStream'; import { PlatformInformation } from '../../../src/platform'; -import { BaseEvent, InstallationProgress } from '../../../src/omnisharp/loggingEvents'; +import { BaseEvent, InstallationStart } from '../../../src/omnisharp/loggingEvents'; import { Files, Binaries, createTestZipAsync } from '../testAssets/CreateTestZip'; chai.use(require("chai-as-promised")); @@ -45,7 +45,7 @@ suite('ZipInstaller', () => { }); test('The folder is unzipped and all the files are present at the expected paths', async () => { - await InstallPackage(zipFileDescriptor, fileDescription, installationPath, [], eventStream); + await InstallZip(zipFileDescriptor, fileDescription, installationPath, [], eventStream); for (let elem of allFiles) { let filePath = path.join(installationPath, elem.path); expect(await util.fileExists(filePath)).to.be.true; @@ -53,9 +53,9 @@ suite('ZipInstaller', () => { }); test('The folder is unzipped and all the expected events are created', async () => { - await InstallPackage(zipFileDescriptor, fileDescription, installationPath, [], eventStream); + await InstallZip(zipFileDescriptor, fileDescription, installationPath, [], eventStream); let eventSequence: BaseEvent[] = [ - new InstallationProgress('installPackages', fileDescription) + new InstallationStart(fileDescription) ]; expect(eventBus).to.be.deep.equal(eventSequence); }); @@ -63,7 +63,7 @@ suite('ZipInstaller', () => { test('The folder is unzipped and the binaries have the expected permissions(except on Windows)', async () => { if (!((await PlatformInformation.GetCurrent()).isWindows())) { let resolvedBinaryPaths = Binaries.map(binary => path.join(installationPath, binary.path)); - await InstallPackage(zipFileDescriptor, fileDescription, installationPath, resolvedBinaryPaths, eventStream); + await InstallZip(zipFileDescriptor, fileDescription, installationPath, resolvedBinaryPaths, eventStream); for (let binaryPath of resolvedBinaryPaths) { expect(await util.fileExists(binaryPath)).to.be.true; let mode = (await fs.stat(binaryPath)).mode; @@ -73,12 +73,12 @@ suite('ZipInstaller', () => { }); test('Error is thrown when the file is not a zip', async () => { - expect(InstallPackage(txtFile.fd, "Text File", installationPath, [], eventStream)).to.be.rejected; + expect(InstallZip(txtFile.fd, "Text File", installationPath, [], eventStream)).to.be.rejected; }); test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file - expect(InstallPackage(0, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; + expect(InstallZip(0, fileDescription, "someRandomPath", [], eventStream)).to.be.rejected; }); teardown(async () => { diff --git a/test/unitTests/logging/CsharpLoggerObserver.test.ts b/test/unitTests/logging/CsharpLoggerObserver.test.ts index a4189e8ca..138d0f30c 100644 --- a/test/unitTests/logging/CsharpLoggerObserver.test.ts +++ b/test/unitTests/logging/CsharpLoggerObserver.test.ts @@ -142,7 +142,7 @@ suite("CsharpLoggerObserver", () => { }); test(`InstallationProgress: Progress message is logged`, () => { - let event = new Event.InstallationProgress("someStage", "somPackage"); + let event = new Event.InstallationStart("somPackage"); observer.post(event); expect(logOutput).to.contain(event.packageDescription); }); diff --git a/test/unitTests/logging/OmnisharpStatusBarObserver.test.ts b/test/unitTests/logging/OmnisharpStatusBarObserver.test.ts index afd03f758..a46536d0f 100644 --- a/test/unitTests/logging/OmnisharpStatusBarObserver.test.ts +++ b/test/unitTests/logging/OmnisharpStatusBarObserver.test.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { StatusBarItem } from '../../../src/vscodeAdapter'; -import { OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnServerError, OmnisharpServerOnStart, OmnisharpServerOnStop, DownloadStart, InstallationProgress, DownloadProgress } from '../../../src/omnisharp/loggingEvents'; +import { OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnServerError, OmnisharpServerOnStart, OmnisharpServerOnStop, DownloadStart, InstallationStart, DownloadProgress } from '../../../src/omnisharp/loggingEvents'; import { expect, should } from 'chai'; import { OmnisharpStatusBarObserver } from '../../../src/observers/OmnisharpStatusBarObserver'; @@ -81,7 +81,7 @@ suite('OmnisharpStatusBarObserver', () => { }); test('InstallationProgress: Text and tooltip are set', () => { - let event = new InstallationProgress("someStage", "somePackage"); + let event = new InstallationStart("somePackage"); observer.post(event); expect(statusBarItem.text).to.contain("Installing packages"); expect(statusBarItem.tooltip).to.contain(event.packageDescription); diff --git a/test/unitTests/testAssets/MockHttpServerRequestOptions.ts b/test/unitTests/testAssets/MockHttpServerRequestHandler.ts similarity index 75% rename from test/unitTests/testAssets/MockHttpServerRequestOptions.ts rename to test/unitTests/testAssets/MockHttpServerRequestHandler.ts index 177b3dd0d..97b81054d 100644 --- a/test/unitTests/testAssets/MockHttpServerRequestOptions.ts +++ b/test/unitTests/testAssets/MockHttpServerRequestHandler.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -export interface MockHttpServerRequestOptions { +export interface MockHttpServerRequestHandler { method: string; path: string; reply: { @@ -14,7 +14,7 @@ export interface MockHttpServerRequestOptions { } -export function getResponseHandlerOptions(method: string, path: string, reply_status: number, reply_headers?: any, reply_body?: any): MockHttpServerRequestOptions { +export function getRequestHandler(method: string, path: string, reply_status: number, reply_headers?: any, reply_body?: any): MockHttpServerRequestHandler { return { method, path, From bde8b49fb970092a464328f78351df6c9b1103a4 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 15:19:55 -0700 Subject: [PATCH 57/64] Downloader test running using free port --- src/omnisharp/OmnisharpDownloader.ts | 3 +- .../unitTests/Packages/FileDownloader.test.ts | 49 ++++++++++++------- .../unitTests/Packages/PackageManager.test.ts | 8 +-- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 6ea1be97a..9dbe6924f 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -43,7 +43,6 @@ export class OmnisharpDownloader { } public async GetLatestVersion(serverUrl: string, latestVersionFileServerPath: string): Promise { - let installationStage = 'getLatestVersionInfoFile'; let description = "Latest Omnisharp Version Information"; let url = `${serverUrl}/${latestVersionFileServerPath}`; let tmpFile: TmpAsset; @@ -54,7 +53,7 @@ export class OmnisharpDownloader { return fs.readFileSync(tmpFile.name, 'utf8'); } catch (error) { - this.eventStream.post(new InstallationFailure(installationStage, error)); + this.eventStream.post(new InstallationFailure('getLatestVersionInfoFile', error)); throw error; } finally { diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 5a38f99fb..af80cb1b7 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -28,6 +28,23 @@ suite("FileDownloader", () => { const networkSettingsProvider = () => new NetworkSettings(undefined, false); const eventStream = new EventStream(); let eventBus: BaseEvent[]; + const getPrimaryURLEvents = () => { + return [ + new DownloadStart(fileDescription), + new DownloadSizeObtained(12), + new DownloadProgress(100, fileDescription), + new DownloadSuccess(' Done!')]; + }; + + const getFallBackURLEvents = () => { + return [ + new DownloadStart(fileDescription), + new DownloadFailure("failed (error code '404')"), + new DownloadFallBack(`${httpsServerUrl}${correctUrlPath}`), + new DownloadSizeObtained(12), + new DownloadProgress(100, fileDescription), + new DownloadSuccess(' Done!')]; + }; eventStream.subscribe((event) => eventBus.push(event)); let server: any; @@ -65,28 +82,18 @@ suite("FileDownloader", () => { description: "Primary url", urlPath: correctUrlPath, fallBackUrlPath: "", - eventsSequence: [ - new DownloadStart(fileDescription), - new DownloadSizeObtained(12), - new DownloadProgress(100, fileDescription), - new DownloadSuccess(' Done!')] + getEventSequence: getPrimaryURLEvents }, { description: "Fallback url", urlPath: errorUrlPath, fallBackUrlPath: correctUrlPath, - eventsSequence: [ - new DownloadStart(fileDescription), - new DownloadFailure("failed (error code '404')"), - new DownloadFallBack(`${httpsServerUrl}${correctUrlPath}`), - new DownloadSizeObtained(12), - new DownloadProgress(100, fileDescription), - new DownloadSuccess(' Done!')] + getEventSequence: getFallBackURLEvents } ].forEach((elem) => { suite(elem.description, () => { test('File is downloaded', async () => { - await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${elem.urlPath}`, `${httpsServerUrl}${elem.fallBackUrlPath}`, eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, getURL(elem.urlPath), getURL(elem.fallBackUrlPath), eventStream, networkSettingsProvider); const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); let text = await fs.readFile(tmpFile.name, 'utf8'); @@ -94,8 +101,8 @@ suite("FileDownloader", () => { }); test('Events are created in the correct order', async () => { - await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${elem.urlPath}`, `${httpsServerUrl}${elem.fallBackUrlPath}`, eventStream, networkSettingsProvider); - expect(eventBus).to.be.deep.equal(elem.eventsSequence); + await DownloadFile(tmpFile.fd, fileDescription, getURL(elem.urlPath), getURL(elem.fallBackUrlPath), eventStream, networkSettingsProvider); + expect(eventBus).to.be.deep.equal(elem.getEventSequence()); }); }); }); @@ -103,7 +110,7 @@ suite("FileDownloader", () => { suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { - await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${redirectUrlPath}`, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, getURL(redirectUrlPath), "", eventStream, networkSettingsProvider); const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); let text = await fs.readFile(tmpFile.name, "utf8"); @@ -113,7 +120,7 @@ suite("FileDownloader", () => { suite('If the response status code is not 301, 302 or 200 then the download fails', () => { test('Error is thrown', async () => { - expect(DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider)).be.rejectedWith(Error); + expect(DownloadFile(tmpFile.fd, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider)).be.rejected; }); test('Download Start and Download Failure events are created', async () => { @@ -122,7 +129,7 @@ suite("FileDownloader", () => { new DownloadFailure("failed (error code '404')") ]; try { - await DownloadFile(tmpFile.fd, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider); } catch (error) { expect(eventBus).to.be.deep.equal(eventsSequence); @@ -132,7 +139,7 @@ suite("FileDownloader", () => { test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file - expect(DownloadFile(0, fileDescription, `${httpsServerUrl}${errorUrlPath}`, "", eventStream, networkSettingsProvider)).to.be.rejected; + expect(DownloadFile(0, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider)).to.be.rejected; }); teardown(async () => { @@ -141,6 +148,10 @@ suite("FileDownloader", () => { tmpFile.dispose(); } }); + + function getURL(urlPath: string) { + return `${httpsServerUrl}${urlPath}`; + } }); diff --git a/test/unitTests/Packages/PackageManager.test.ts b/test/unitTests/Packages/PackageManager.test.ts index ae00b1bd0..bf0eaab42 100644 --- a/test/unitTests/Packages/PackageManager.test.ts +++ b/test/unitTests/Packages/PackageManager.test.ts @@ -51,7 +51,6 @@ suite("Package Manager", () => { key: await fs.readFile("test/unitTests/testAssets/private.pem"), cert: await fs.readFile("test/unitTests/testAssets/public.pem") }); - }); setup(async () => { @@ -71,13 +70,10 @@ suite("Package Manager", () => { testDirPath = tmpSourceDir.name + "/test.zip"; await createTestZipAsync(testDirPath, allFiles); await new Promise(resolve => server.start(resolve)); //start the server - let buffer: any; - let stat = await fs.stat(testDirPath); - buffer = await fs.readFile(testDirPath); server.on(getRequestHandler('GET', '/package', 200, { "content-type": "application/zip", - "content-length": stat.size - }, buffer)); + "content-length": (await fs.stat(testDirPath)).size + }, await fs.readFile(testDirPath))); }); test("Downloads the package and installs at the specified path", async () => { From 7abd6a65a34f8ee6fb684f44cbd4fc53bcf54957 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 15:31:56 -0700 Subject: [PATCH 58/64] Nits --- src/omnisharp/OmnisharpDownloader.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 9dbe6924f..794bc11d9 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -17,7 +17,7 @@ import { ResolveFilePaths } from '../packageManager/PackageFilePathResolver'; export class OmnisharpDownloader { public constructor( - private provider: NetworkSettingsProvider, + private networkSettingsProvider: NetworkSettingsProvider, private eventStream: EventStream, private packageJSON: any, private platformInfo: PlatformInformation) { @@ -33,7 +33,7 @@ export class OmnisharpDownloader { let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath); packages.forEach(pkg => ResolveFilePaths(pkg)); installationStage = 'downloadAndInstallPackages'; - await DownloadAndInstallPackages(packages, this.provider, this.platformInfo, this.eventStream); + await DownloadAndInstallPackages(packages, this.networkSettingsProvider, this.platformInfo, this.eventStream); this.eventStream.post(new InstallationSuccess()); } catch (error) { @@ -49,7 +49,7 @@ export class OmnisharpDownloader { try { this.eventStream.post(new LatestBuildDownloadStart()); tmpFile = await CreateTmpFile(); - await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.provider); + await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.networkSettingsProvider); return fs.readFileSync(tmpFile.name, 'utf8'); } catch (error) { From 3e323afae1a274d79b91ca5920a3b2c02832dd86 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 18:19:09 -0700 Subject: [PATCH 59/64] clean up --- src/CSharpExtDownloader.ts | 26 +++++++++---------- src/NetworkSettings.ts | 3 +-- src/packageManager/PackageFilePathResolver.ts | 7 +++-- src/packageManager/PackageFilterer.ts | 2 +- tasks/offlinePackagingTasks.ts | 4 +-- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts index 8a3611e9e..03c738878 100644 --- a/src/CSharpExtDownloader.ts +++ b/src/CSharpExtDownloader.ts @@ -18,7 +18,7 @@ import { ResolveFilePaths } from './packageManager/PackageFilePathResolver'; export class CSharpExtDownloader { public constructor( - private provider: NetworkSettingsProvider, + private networkSettingsProvider: NetworkSettingsProvider, private eventStream: EventStream, private packageJSON: any, private platformInfo: PlatformInformation) { @@ -26,40 +26,38 @@ export class CSharpExtDownloader { public async installRuntimeDependencies(): Promise { this.eventStream.post(new PackageInstallation("C# dependencies")); - let success = false; - let installationStage = ''; + let installationStage = 'touchBeginFile'; try { - await util.touchInstallFile(util.InstallFileType.Begin); // Display platform information and RID this.eventStream.post(new LogPlatformInfo(this.platformInfo)); - let runTimeDependencies = this.GetRunTimeDependenciesPackages(); + let runTimeDependencies = GetRunTimeDependenciesPackages(this.packageJSON); runTimeDependencies.forEach(pkg => ResolveFilePaths(pkg)); installationStage = 'downloadAndInstallPackages'; - await DownloadAndInstallPackages(runTimeDependencies, this.provider, this.platformInfo, this.eventStream); + await DownloadAndInstallPackages(runTimeDependencies, this.networkSettingsProvider, this.platformInfo, this.eventStream); installationStage = 'touchLockFile'; await util.touchInstallFile(util.InstallFileType.Lock); - success = true; this.eventStream.post(new InstallationSuccess()); + return true; } catch (error) { this.eventStream.post(new InstallationFailure(installationStage, error)); + return false; } finally { try { util.deleteInstallFile(util.InstallFileType.Begin); } catch (error) { } - return success; } } +} - private GetRunTimeDependenciesPackages(): Package[] { - if (this.packageJSON.runtimeDependencies) { - return JSON.parse(JSON.stringify(this.packageJSON.runtimeDependencies)); - } - - throw new Error("No runtime dependencies found"); +export function GetRunTimeDependenciesPackages(packageJSON: any): Package[] { + if (packageJSON.runtimeDependencies) { + return JSON.parse(JSON.stringify(packageJSON.runtimeDependencies)); } + + throw new Error("No runtime dependencies found"); } \ No newline at end of file diff --git a/src/NetworkSettings.ts b/src/NetworkSettings.ts index 2bbea1674..b3be32f04 100644 --- a/src/NetworkSettings.ts +++ b/src/NetworkSettings.ts @@ -11,11 +11,10 @@ export default class NetworkSettings { } export interface NetworkSettingsProvider { - //to do: make this async (): NetworkSettings; } -export function vscodeNetworkSettingsProvider(vscode: vscode): NetworkSettingsProvider { +export async function vscodeNetworkSettingsProvider(vscode: vscode): Promise { return () => { const config = vscode.workspace.getConfiguration(); const proxy = config.get('http.proxy'); diff --git a/src/packageManager/PackageFilePathResolver.ts b/src/packageManager/PackageFilePathResolver.ts index 9d5d409de..7c1fac687 100644 --- a/src/packageManager/PackageFilePathResolver.ts +++ b/src/packageManager/PackageFilePathResolver.ts @@ -17,14 +17,13 @@ export function ResolvePackageTestPath(pkg: Package): string { if (pkg.installTestPath) { return path.resolve(util.getExtensionPath(), pkg.installTestPath); } - else { - return null; - } + + return null; } function ResolvePackageBinaries(pkg: Package) { if (pkg.binaries) { - return pkg.binaries.map(value => path.resolve(ResolveBaseInstallPath(pkg), value)); + return pkg.binaries.map(value => path.resolve(ResolveBaseInstallPath(pkg), value)); } return null; diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index 5e262f9c2..dc7d5513d 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -43,6 +43,6 @@ async function filterAlreadyInstalledPackages(packages: Package[]): Promise { del.sync(vscodeignorePath); @@ -94,7 +94,7 @@ async function install(platformInfo: PlatformInformation, packageJSON: any) { let stdoutObserver = new CsharpLoggerObserver(logger); eventStream.subscribe(stdoutObserver.post); const debuggerUtil = new debugUtil.CoreClrDebugUtil(path.resolve('.')); - let runTimeDependencies = JSON.parse(JSON.stringify(packageJSON.runtimeDependencies)); + let runTimeDependencies = GetRunTimeDependenciesPackages(packageJSON); let provider = () => new NetworkSettings(undefined, undefined); await DownloadAndInstallPackages(runTimeDependencies, provider, platformInfo, eventStream); await debugUtil.CoreClrDebugUtil.writeEmptyFile(debuggerUtil.installCompleteFilePath()); From 41b7b87a14d108c4fc35694d3293b120f2a4863f Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 23 Apr 2018 19:16:51 -0700 Subject: [PATCH 60/64] Making fallback url optional --- src/NetworkSettings.ts | 2 +- src/omnisharp/OmnisharpDownloader.ts | 4 ++-- src/packageManager/FileDownloader.ts | 2 +- src/packageManager/PackageManager.ts | 4 ++-- test/unitTests/Packages/FileDownloader.test.ts | 12 ++++++------ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/NetworkSettings.ts b/src/NetworkSettings.ts index b3be32f04..1ae77fe0b 100644 --- a/src/NetworkSettings.ts +++ b/src/NetworkSettings.ts @@ -14,7 +14,7 @@ export interface NetworkSettingsProvider { (): NetworkSettings; } -export async function vscodeNetworkSettingsProvider(vscode: vscode): Promise { +export function vscodeNetworkSettingsProvider(vscode: vscode): NetworkSettingsProvider { return () => { const config = vscode.workspace.getConfiguration(); const proxy = config.get('http.proxy'); diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts index 794bc11d9..ba40cfe18 100644 --- a/src/omnisharp/OmnisharpDownloader.ts +++ b/src/omnisharp/OmnisharpDownloader.ts @@ -24,7 +24,7 @@ export class OmnisharpDownloader { } public async DownloadAndInstallOmnisharp(version: string, serverUrl: string, installPath: string) { - this.eventStream.post(new PackageInstallation(`Omnisharp Version = ${version}`)); + this.eventStream.post(new PackageInstallation(`OmniSharp Version = ${version}`)); let installationStage = ''; try { @@ -49,7 +49,7 @@ export class OmnisharpDownloader { try { this.eventStream.post(new LatestBuildDownloadStart()); tmpFile = await CreateTmpFile(); - await DownloadFile(tmpFile.fd, description, url, "", this.eventStream, this.networkSettingsProvider); + await DownloadFile(tmpFile.fd, description, this.eventStream, this.networkSettingsProvider, url); return fs.readFileSync(tmpFile.name, 'utf8'); } catch (error) { diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index a2537d7b1..9cfe3eecd 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -13,7 +13,7 @@ import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; -export async function DownloadFile(destinationFileDescriptor: number, description: string, url: string, fallbackUrl: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider){ +export async function DownloadFile(destinationFileDescriptor: number, description: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider, url: string, fallbackUrl?: string,){ eventStream.post(new DownloadStart(description)); try { diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index ed0a72696..257ee6853 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -21,7 +21,7 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: for (let pkg of filteredPackages) { try { tmpFile = await CreateTmpFile(); - await DownloadFile(tmpFile.fd, pkg.description, pkg.url, pkg.fallbackUrl, eventStream, provider); + await DownloadFile(tmpFile.fd, pkg.description, eventStream, provider, pkg.url, pkg.fallbackUrl); await InstallZip(tmpFile.fd, pkg.description, pkg.installPath, pkg.binaries, eventStream); } catch (error) { @@ -35,7 +35,7 @@ export async function DownloadAndInstallPackages(packages: Package[], provider: finally { //clean the temporary file if (tmpFile) { - await tmpFile.dispose(); + tmpFile.dispose(); } } } diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index af80cb1b7..5c77b8926 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -93,7 +93,7 @@ suite("FileDownloader", () => { ].forEach((elem) => { suite(elem.description, () => { test('File is downloaded', async () => { - await DownloadFile(tmpFile.fd, fileDescription, getURL(elem.urlPath), getURL(elem.fallBackUrlPath), eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, eventStream, networkSettingsProvider, getURL(elem.urlPath), getURL(elem.fallBackUrlPath)); const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); let text = await fs.readFile(tmpFile.name, 'utf8'); @@ -101,7 +101,7 @@ suite("FileDownloader", () => { }); test('Events are created in the correct order', async () => { - await DownloadFile(tmpFile.fd, fileDescription, getURL(elem.urlPath), getURL(elem.fallBackUrlPath), eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, eventStream, networkSettingsProvider, getURL(elem.urlPath), getURL(elem.fallBackUrlPath)); expect(eventBus).to.be.deep.equal(elem.getEventSequence()); }); }); @@ -110,7 +110,7 @@ suite("FileDownloader", () => { suite('If the response status Code is 301, redirect occurs and the download succeeds', () => { test('File is downloaded from the redirect url', async () => { - await DownloadFile(tmpFile.fd, fileDescription, getURL(redirectUrlPath), "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, eventStream, networkSettingsProvider, getURL(redirectUrlPath)); const stats = await fs.stat(tmpFile.name); expect(stats.size).to.not.equal(0); let text = await fs.readFile(tmpFile.name, "utf8"); @@ -120,7 +120,7 @@ suite("FileDownloader", () => { suite('If the response status code is not 301, 302 or 200 then the download fails', () => { test('Error is thrown', async () => { - expect(DownloadFile(tmpFile.fd, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider)).be.rejected; + expect(DownloadFile(tmpFile.fd, fileDescription, eventStream, networkSettingsProvider, getURL(errorUrlPath))).be.rejected; }); test('Download Start and Download Failure events are created', async () => { @@ -129,7 +129,7 @@ suite("FileDownloader", () => { new DownloadFailure("failed (error code '404')") ]; try { - await DownloadFile(tmpFile.fd, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider); + await DownloadFile(tmpFile.fd, fileDescription, eventStream, networkSettingsProvider, getURL(errorUrlPath)); } catch (error) { expect(eventBus).to.be.deep.equal(eventsSequence); @@ -139,7 +139,7 @@ suite("FileDownloader", () => { test('Error is thrown on invalid input file', async () => { //fd=0 means there is no file - expect(DownloadFile(0, fileDescription, getURL(errorUrlPath), "", eventStream, networkSettingsProvider)).to.be.rejected; + expect(DownloadFile(0, fileDescription, eventStream, networkSettingsProvider, getURL(errorUrlPath))).to.be.rejected; }); teardown(async () => { From 28fab5a1d8e533e068c581fa3f4e25768613a90f Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 24 Apr 2018 10:43:12 -0700 Subject: [PATCH 61/64] using else if --- src/packageManager/FileDownloader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index 9cfe3eecd..6a3d18f9e 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -64,7 +64,7 @@ async function downloadFile(fd: number, description: string, urlString: string, return resolve(downloadFile(fd, description, response.headers.location, eventStream, networkSettingsProvider)); } - if (response.statusCode != 200) { + else if (response.statusCode != 200) { // Download failed - print error message eventStream.post(new DownloadFailure(`failed (error code '${response.statusCode}')`)); return reject(new NestedError(response.statusCode.toString())); From 45300acf530132895a46ceb4bd1dfd6b4371d150 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 24 Apr 2018 13:54:12 -0700 Subject: [PATCH 62/64] PR feedback --- src/NestedError.ts | 10 ++++++++++ src/observers/BaseChannelObserver.ts | 2 +- src/observers/BaseLoggerObserver.ts | 2 +- src/packageManager/FileDownloader.ts | 4 ++-- src/packageManager/Package.ts | 17 ----------------- src/packageManager/PackageError.ts | 16 ++++++++++++++++ src/packageManager/PackageManager.ts | 10 ++++++---- test/unitTests/Packages/FileDownloader.test.ts | 1 - 8 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 src/NestedError.ts create mode 100644 src/packageManager/PackageError.ts diff --git a/src/NestedError.ts b/src/NestedError.ts new file mode 100644 index 000000000..26de1e87d --- /dev/null +++ b/src/NestedError.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export class NestedError extends Error { + constructor(public message: string, public err: Error = null) { + super(message); + } +} \ No newline at end of file diff --git a/src/observers/BaseChannelObserver.ts b/src/observers/BaseChannelObserver.ts index 2dde4663b..cdebd5b09 100644 --- a/src/observers/BaseChannelObserver.ts +++ b/src/observers/BaseChannelObserver.ts @@ -14,7 +14,7 @@ export abstract class BaseChannelObserver { abstract post: (event: BaseEvent) => void; public showChannel(preserveFocusOrColumn?: boolean) { - this.channel.show(preserveFocusOrColumn as boolean); + this.channel.show(preserveFocusOrColumn); } public clearChannel() { diff --git a/src/observers/BaseLoggerObserver.ts b/src/observers/BaseLoggerObserver.ts index d40a4c80b..cedf31e7a 100644 --- a/src/observers/BaseLoggerObserver.ts +++ b/src/observers/BaseLoggerObserver.ts @@ -11,7 +11,7 @@ export abstract class BaseLoggerObserver { public logger: Logger; constructor(channel: vscode.OutputChannel | Logger) { if (channel instanceof Logger) { - this.logger = channel as Logger; + this.logger = channel; } else { this.logger = new Logger((message) => channel.append(message)); diff --git a/src/packageManager/FileDownloader.ts b/src/packageManager/FileDownloader.ts index 6a3d18f9e..0541f8e46 100644 --- a/src/packageManager/FileDownloader.ts +++ b/src/packageManager/FileDownloader.ts @@ -8,12 +8,12 @@ import * as util from '../common'; import * as fs from 'fs'; import { EventStream } from "../EventStream"; import { DownloadSuccess, DownloadStart, DownloadFallBack, DownloadFailure, DownloadProgress, DownloadSizeObtained } from "../omnisharp/loggingEvents"; -import { NestedError } from "./Package"; +import { NestedError } from "../NestedError"; import { parse as parseUrl } from 'url'; import { getProxyAgent } from './proxy'; import { NetworkSettingsProvider } from '../NetworkSettings'; -export async function DownloadFile(destinationFileDescriptor: number, description: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider, url: string, fallbackUrl?: string,){ +export async function DownloadFile(destinationFileDescriptor: number, description: string, eventStream: EventStream, networkSettingsProvider: NetworkSettingsProvider, url: string, fallbackUrl?: string){ eventStream.post(new DownloadStart(description)); try { diff --git a/src/packageManager/Package.ts b/src/packageManager/Package.ts index 1ab260df9..f941d5578 100644 --- a/src/packageManager/Package.ts +++ b/src/packageManager/Package.ts @@ -15,20 +15,3 @@ export interface Package { // Path to use to test if the package has already been installed installTestPath?: string; } - -export class NestedError extends Error { - constructor(public message: string, public err: any = null) { - super(message); - } -} - -export class PackageError extends NestedError { - // Do not put PII (personally identifiable information) in the 'message' field as it will be logged to telemetry - constructor(public message: string, - public pkg: Package = null, - public innerError: any = null) { - super(message, innerError); - } -} - - diff --git a/src/packageManager/PackageError.ts b/src/packageManager/PackageError.ts new file mode 100644 index 000000000..69423f0e9 --- /dev/null +++ b/src/packageManager/PackageError.ts @@ -0,0 +1,16 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { NestedError } from '../NestedError'; +import { Package } from './Package'; + +export class PackageError extends NestedError { + // Do not put PII (personally identifiable information) in the 'message' field as it will be logged to telemetry + constructor(public message: string, + public pkg: Package = null, + public innerError: any = null) { + super(message, innerError); + } +} \ No newline at end of file diff --git a/src/packageManager/PackageManager.ts b/src/packageManager/PackageManager.ts index 257ee6853..64b229b9f 100644 --- a/src/packageManager/PackageManager.ts +++ b/src/packageManager/PackageManager.ts @@ -4,7 +4,9 @@ *--------------------------------------------------------------------------------------------*/ import { PlatformInformation } from "../platform"; -import { Package, PackageError, NestedError } from './Package'; +import { Package } from './Package'; +import { PackageError } from './PackageError'; +import { NestedError } from "../NestedError"; import { DownloadFile } from './FileDownloader'; import { InstallZip } from './ZipInstaller'; import { EventStream } from '../EventStream'; @@ -12,12 +14,12 @@ import { NetworkSettingsProvider } from "../NetworkSettings"; import { filterPackages } from "./PackageFilterer"; import { CreateTmpFile, TmpAsset } from "../CreateTmpAsset"; -//Package manager needs a list of packages to be filtered based on platformInfo then download and install them -//Note that the packages that this component will install needs absolute paths for the installPath, intsallTestPath and the binaries +// Package manager needs a list of packages to be filtered based on platformInfo then download and install them +// Note that the packages that this component will install needs absolute paths for the installPath, intsallTestPath and the binaries export async function DownloadAndInstallPackages(packages: Package[], provider: NetworkSettingsProvider, platformInfo: PlatformInformation, eventStream: EventStream) { let filteredPackages = await filterPackages(packages, platformInfo); - let tmpFile: TmpAsset; if (filteredPackages) { + let tmpFile: TmpAsset; for (let pkg of filteredPackages) { try { tmpFile = await CreateTmpFile(); diff --git a/test/unitTests/Packages/FileDownloader.test.ts b/test/unitTests/Packages/FileDownloader.test.ts index 5c77b8926..92a1a17b9 100644 --- a/test/unitTests/Packages/FileDownloader.test.ts +++ b/test/unitTests/Packages/FileDownloader.test.ts @@ -19,7 +19,6 @@ chai.use(require("chai-as-promised")); chai.use(require('chai-arrays')); const expect = chai.expect; -//to do:look into http url thing suite("FileDownloader", () => { const fileDescription = "Test file"; const correctUrlPath = `/resource`; From 573ca83c5b7314d148f4997b993d028c043cf38f Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 24 Apr 2018 13:57:56 -0700 Subject: [PATCH 63/64] Package Error --- src/CreateTmpAsset.ts | 2 +- src/observers/CsharpLoggerObserver.ts | 3 ++- src/observers/TelemetryObserver.ts | 2 +- src/packageManager/PackageFilterer.ts | 3 ++- src/packageManager/ZipInstaller.ts | 2 +- test/unitTests/logging/CsharpLoggerObserver.test.ts | 2 +- test/unitTests/logging/TelemetryObserver.test.ts | 3 ++- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/CreateTmpAsset.ts b/src/CreateTmpAsset.ts index fb13921b2..adfd93ffb 100644 --- a/src/CreateTmpAsset.ts +++ b/src/CreateTmpAsset.ts @@ -3,8 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as tmp from 'tmp'; -import { NestedError } from './packageManager/Package'; import { rimraf } from 'async-file'; +import { NestedError } from './NestedError'; export async function CreateTmpFile(): Promise { const tmpFile = await new Promise((resolve, reject) => { diff --git a/src/observers/CsharpLoggerObserver.ts b/src/observers/CsharpLoggerObserver.ts index b67044868..35fd4922d 100644 --- a/src/observers/CsharpLoggerObserver.ts +++ b/src/observers/CsharpLoggerObserver.ts @@ -5,7 +5,8 @@ import { BaseLoggerObserver } from "./BaseLoggerObserver"; import * as Event from "../omnisharp/loggingEvents"; -import { PackageError } from "../packageManager/Package"; +import { PackageError } from "../packageManager/PackageError"; + export class CsharpLoggerObserver extends BaseLoggerObserver { private dots: number; diff --git a/src/observers/TelemetryObserver.ts b/src/observers/TelemetryObserver.ts index 8911d10a7..6453df9aa 100644 --- a/src/observers/TelemetryObserver.ts +++ b/src/observers/TelemetryObserver.ts @@ -5,7 +5,7 @@ import { PlatformInformation } from "../platform"; import { BaseEvent, PackageInstallation, InstallationFailure, InstallationSuccess, OmnisharpDelayTrackerEventMeasures, OmnisharpStart, TestExecutionCountReport, TelemetryEventWithMeasures } from "../omnisharp/loggingEvents"; -import { PackageError } from "../packageManager/Package"; +import { PackageError } from "../packageManager/PackageError"; export interface ITelemetryReporter { sendTelemetryEvent(eventName: string, properties?: { [key: string]: string }, measures?: { [key: string]: number }): void; diff --git a/src/packageManager/PackageFilterer.ts b/src/packageManager/PackageFilterer.ts index dc7d5513d..36ead88ac 100644 --- a/src/packageManager/PackageFilterer.ts +++ b/src/packageManager/PackageFilterer.ts @@ -3,10 +3,11 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Package, PackageError } from "./Package"; +import { Package } from "./Package"; import { PlatformInformation } from "../platform"; import * as util from '../common'; import { ResolvePackageTestPath } from "./PackageFilePathResolver"; +import { PackageError } from "./PackageError"; const { filterAsync } = require('node-filter-async'); diff --git a/src/packageManager/ZipInstaller.ts b/src/packageManager/ZipInstaller.ts index b31311e21..3907254be 100644 --- a/src/packageManager/ZipInstaller.ts +++ b/src/packageManager/ZipInstaller.ts @@ -9,7 +9,7 @@ import * as path from 'path'; import * as yauzl from 'yauzl'; import { EventStream } from "../EventStream"; import { InstallationStart } from "../omnisharp/loggingEvents"; -import { NestedError } from './Package'; +import { NestedError } from '../NestedError'; export async function InstallZip(sourceFileDescriptor: number, description: string, destinationInstallPath: string, binaries: string[], eventStream: EventStream): Promise { eventStream.post(new InstallationStart(description)); diff --git a/test/unitTests/logging/CsharpLoggerObserver.test.ts b/test/unitTests/logging/CsharpLoggerObserver.test.ts index 138d0f30c..678af6f57 100644 --- a/test/unitTests/logging/CsharpLoggerObserver.test.ts +++ b/test/unitTests/logging/CsharpLoggerObserver.test.ts @@ -8,7 +8,7 @@ import { getNullChannel } from '../testAssets/Fakes'; import { CsharpLoggerObserver } from '../../../src/observers/CsharpLoggerObserver'; import { PlatformInformation } from '../../../src/platform'; import * as Event from '../../../src/omnisharp/loggingEvents'; -import { PackageError } from '../../../src/packageManager/Package'; +import { PackageError } from '../../../src/packageManager/PackageError'; suite("CsharpLoggerObserver", () => { suiteSetup(() => should()); diff --git a/test/unitTests/logging/TelemetryObserver.test.ts b/test/unitTests/logging/TelemetryObserver.test.ts index ff7743739..e7398fab5 100644 --- a/test/unitTests/logging/TelemetryObserver.test.ts +++ b/test/unitTests/logging/TelemetryObserver.test.ts @@ -7,7 +7,8 @@ import { TelemetryObserver } from '../../../src/observers/TelemetryObserver'; import { PlatformInformation } from '../../../src/platform'; import { PackageInstallation, InstallationFailure, InstallationSuccess, TestExecutionCountReport, TelemetryEventWithMeasures, OmnisharpDelayTrackerEventMeasures, OmnisharpStart } from '../../../src/omnisharp/loggingEvents'; import { getNullTelemetryReporter } from '../testAssets/Fakes'; -import { PackageError, Package } from '../../../src/packageManager/Package'; +import { Package } from '../../../src/packageManager/Package'; +import { PackageError } from '../../../src/packageManager/PackageError'; const chai = require('chai'); chai.use(require('chai-arrays')); From 87fff8b58fc3e411370f7f03b0e3ae55a9008017 Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 24 Apr 2018 14:20:37 -0700 Subject: [PATCH 64/64] dispose if present --- test/unitTests/Packages/PackageFilterer.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unitTests/Packages/PackageFilterer.test.ts b/test/unitTests/Packages/PackageFilterer.test.ts index f18c84e7a..d17e9024c 100644 --- a/test/unitTests/Packages/PackageFilterer.test.ts +++ b/test/unitTests/Packages/PackageFilterer.test.ts @@ -96,6 +96,8 @@ suite('PackageFilterer', () => { }); teardown(() => { - tmpFile.dispose(); + if (tmpFile) { + tmpFile.dispose(); + } }); }); \ No newline at end of file