Skip to content

Commit

Permalink
Changes to IntelliCode and engine downloads (#1715)
Browse files Browse the repository at this point in the history
* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* #1096 The if statement is automatically formatted incorrectly

* Merge fix

* Add more tests

* More tests

* Typo

* Test

* Also better handle multiline arguments

* Add a couple missing periods

[skip ci]

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Merge fix

* #1257 On type formatting errors for args and kwargs

* Handle f-strings

* Stop importing from test code

* #1308 Single line statements leading to an indentation on the next line

* #726 editing python after inline if statement invalid indent

* Undo change

* Move constant

* Harden LS startup error checks

* #1364 Intellisense doesn't work after specific const string

* Telemetry for the analysis enging

* PR feedback

* Fix typo

* Test baseline update

* Jedi 0.12

* Priority to goto_defition

* News

* Replace unzip

* Linux flavors + test

* Grammar check

* Grammar test

* Test baselines

* Add news

* Pin dependency

[skip ci]

* Specify markdown as preferable format

* Improve function argument detection

* Specify markdown

* Pythia setting

* Baseline updates

* Baseline update

* Improve startup

* Handle missing interpreter better

* Handle interpreter change

* Delete old file

* Fix LS startup time reporting

* Remove Async suffix from IFileSystem

* Remove Pythia

* Remove pre-packaged MSIL

* Exe name on Unix

* Plain linux

* Fix casing

* Fix message

* Update PTVS engine activation steps
  • Loading branch information
Mikhail Arkhipov authored May 21, 2018
1 parent 35fe3c0 commit ca657e0
Show file tree
Hide file tree
Showing 8 changed files with 27 additions and 109 deletions.
1 change: 1 addition & 0 deletions CONTRIBUTING - PYTHON_ANALYSIS.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ Visual Studio 2017:
3. Binaries arrive in *Python/BuildOutput/VsCode/raw*
4. Delete contents of the *analysis* folder in the Python Extension folder
5. Copy *.dll, *.pdb, *.json fron *Python/BuildOutput/VsCode/raw* to *analysis*
6. In VS Code set setting *python.downloadCodeAnalysis* to *false*

### Debugging code in Python Extension to VS Code
Folow regular TypeScript debugging steps
Expand Down
6 changes: 0 additions & 6 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -1232,12 +1232,6 @@
"description": "Whether to install Python modules globally when not using an environment.",
"scope": "resource"
},
"python.pythiaEnabled": {
"type": "boolean",
"default": true,
"description": "Enables AI-driven additions to the completion list. Does not apply to Jedi.",
"scope": "resource"
},
"python.jediEnabled": {
"type": "boolean",
"default": true,
Expand Down
41 changes: 7 additions & 34 deletions src/client/activation/analysis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import { IApplicationShell } from '../common/application/types';
import { isTestExecution, STANDARD_OUTPUT_CHANNEL } from '../common/constants';
import { createDeferred, Deferred } from '../common/helpers';
import { IFileSystem, IPlatformService } from '../common/platform/types';
import { IProcessServiceFactory } from '../common/process/types';
import { StopWatch } from '../common/stopWatch';
import { IConfigurationService, IOutputChannel, IPythonSettings } from '../common/types';
import { IEnvironmentVariablesProvider } from '../common/variables/types';
Expand Down Expand Up @@ -103,32 +102,19 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
// Determine if we are running MSIL/Universal via dotnet or self-contained app.
const mscorlib = path.join(context.extensionPath, analysisEngineFolder, 'mscorlib.dll');
const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
let downloadPackage = false;

const reporter = getTelemetryReporter();
reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ENABLED);

await this.checkPythiaModel(context, downloader);

if (!await this.fs.fileExists(mscorlib)) {
// Depends on .NET Runtime or SDK
const settings = this.configuration.getSettings();
if (!settings.downloadCodeAnalysis) {
// Depends on .NET Runtime or SDK. Typically development-only case.
this.languageClient = this.createSimpleLanguageClient(context, clientOptions);
try {
await this.tryStartLanguageClient(context, this.languageClient);
return true;
} catch (ex) {
if (await this.isDotNetInstalled()) {
this.appShell.showErrorMessage(`.NET Runtime appears to be installed but the language server did not start. Error ${ex}`);
reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ERROR, { error: 'Failed to start (MSIL)' });
return false;
}
// No .NET Runtime, no mscorlib - need to download self-contained package.
downloadPackage = true;
}
await this.tryStartLanguageClient(context, this.languageClient);
return true;
}

if (downloadPackage) {
this.appShell.showWarningMessage('.NET Runtime is not found, platform-specific Python Analysis Engine will be downloaded.');
if (!await this.fs.fileExists(mscorlib)) {
await downloader.downloadAnalysisEngine(context);
reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_DOWNLOADED);
}
Expand Down Expand Up @@ -254,22 +240,9 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
maxDocumentationTextLength: 0
},
asyncStartup: true,
pythiaEnabled: settings.pythiaEnabled,
intelliCodeEnabled: settings.intelliCodeEnabled,
testEnvironment: isTestExecution()
}
};
}

private async isDotNetInstalled(): Promise<boolean> {
const ps = await this.services.get<IProcessServiceFactory>(IProcessServiceFactory).create();
const result = await ps.exec('dotnet', ['--version']).catch(() => { return { stdout: '' }; });
return result.stdout.trim().startsWith('2.');
}

private async checkPythiaModel(context: ExtensionContext, downloader: AnalysisEngineDownloader): Promise<void> {
const settings = this.configuration.getSettings();
if (settings.pythiaEnabled) {
await downloader.downloadPythiaModel(context);
}
}
}
8 changes: 1 addition & 7 deletions src/client/activation/analysisEngineHashes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,4 @@
export const analysis_engine_win_x86_sha512 = 'win-x86';
export const analysis_engine_win_x64_sha512 = 'win-x64';
export const analysis_engine_osx_x64_sha512 = 'osx-x64';
export const analysis_engine_centos_x64_sha512 = 'centos-x64';
export const analysis_engine_debian_x64_sha512 = 'debian-x64';
export const analysis_engine_fedora_x64_sha512 = 'fedora-x64';
export const analysis_engine_ol_x64_sha512 = 'ol-x64';
export const analysis_engine_opensuse_x64_sha512 = 'opensuse-x64';
export const analysis_engine_rhel_x64_sha512 = 'rhel-x64';
export const analysis_engine_ubuntu_x64_sha512 = 'ubuntu-x64';
export const analysis_engine_linux_x64_sha512 = 'linux-x64';
15 changes: 7 additions & 8 deletions src/client/activation/downloader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ import { PlatformData } from './platformData';
const StreamZip = require('node-stream-zip');

const downloadUriPrefix = 'https://pvsc.blob.core.windows.net/python-analysis';
const downloadBaseFileName = 'python-analysis-vscode';
const downloadBaseFileName = 'Python-Analysis-VSCode';
const downloadVersion = '0.1.0';
const downloadFileExtension = '.nupkg';
const pythiaModelName = 'model-sequence.json.gz';
const modelName = 'model-sequence.json.gz';

export class AnalysisEngineDownloader {
private readonly output: OutputChannel;
Expand Down Expand Up @@ -56,16 +56,16 @@ export class AnalysisEngineDownloader {
}
}

public async downloadPythiaModel(context: ExtensionContext): Promise<void> {
public async downloadIntelliCodeModel(context: ExtensionContext): Promise<void> {
const modelFolder = path.join(context.extensionPath, 'analysis', 'Pythia', 'model');
const localPath = path.join(modelFolder, pythiaModelName);
const localPath = path.join(modelFolder, modelName);
if (await this.fs.fileExists(localPath)) {
return;
}

let localTempFilePath = '';
try {
localTempFilePath = await this.downloadFile(downloadUriPrefix, pythiaModelName, 'Downloading IntelliSense Model File... ');
localTempFilePath = await this.downloadFile(downloadUriPrefix, modelName, 'Downloading IntelliCode Model File... ');
await this.fs.createDirectory(modelFolder);
await this.fs.copyFile(localTempFilePath, localPath);
} catch (err) {
Expand Down Expand Up @@ -129,11 +129,10 @@ export class AnalysisEngineDownloader {
if (!await verifier.verifyHash(filePath, platformString, await this.platformData.getExpectedHash())) {
throw new Error('Hash of the downloaded file does not match.');
}
this.output.append('valid.');
this.output.appendLine('valid.');
}

private async unpackArchive(extensionPath: string, tempFilePath: string): Promise<void> {
this.output.appendLine('');
this.output.append('Unpacking archive... ');

const installFolder = path.join(extensionPath, this.engineFolder);
Expand Down Expand Up @@ -170,12 +169,12 @@ export class AnalysisEngineDownloader {
});
return deferred.promise;
});
this.output.append('done.');

// Set file to executable
if (!this.platform.isWindows) {
const executablePath = path.join(installFolder, this.platformData.getEngineExecutableName());
fileSystem.chmodSync(executablePath, '0764'); // -rwxrw-r--
}
this.output.appendLine('done.');
}
}
56 changes: 5 additions & 51 deletions src/client/activation/platformData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,14 @@

import { IFileSystem, IPlatformService } from '../common/platform/types';
import {
analysis_engine_centos_x64_sha512,
analysis_engine_debian_x64_sha512,
analysis_engine_fedora_x64_sha512,
analysis_engine_ol_x64_sha512,
analysis_engine_opensuse_x64_sha512,
analysis_engine_linux_x64_sha512,
analysis_engine_osx_x64_sha512,
analysis_engine_rhel_x64_sha512,
analysis_engine_ubuntu_x64_sha512,
analysis_engine_win_x64_sha512,
analysis_engine_win_x86_sha512
} from './analysisEngineHashes';

// '/etc/os-release', ID=flavor
const supportedLinuxFlavors = [
'centos',
'debian',
'fedora',
'ol',
'opensuse',
'rhel',
'ubuntu'
];

export class PlatformData {
constructor(private platform: IPlatformService, private fs: IFileSystem) { }
constructor(private platform: IPlatformService, fs: IFileSystem) { }
public async getPlatformName(): Promise<string> {
if (this.platform.isWindows) {
return this.platform.is64bit ? 'win-x64' : 'win-x86';
Expand All @@ -39,14 +22,7 @@ export class PlatformData {
if (!this.platform.is64bit) {
throw new Error('Python Analysis Engine does not support 32-bit Linux.');
}
const linuxFlavor = await this.getLinuxFlavor();
if (linuxFlavor.length === 0) {
throw new Error('Unable to determine Linux flavor from /etc/os-release.');
}
if (supportedLinuxFlavors.indexOf(linuxFlavor) < 0) {
throw new Error(`${linuxFlavor} is not supported.`);
}
return `${linuxFlavor}-x64`;
return 'linux-x64';
}
throw new Error('Unknown OS platform.');
}
Expand All @@ -58,7 +34,7 @@ export class PlatformData {
public getEngineExecutableName(): string {
return this.platform.isWindows
? 'Microsoft.PythonTools.VsCode.exe'
: 'Microsoft.PythonTools.VsCode';
: 'Microsoft.PythonTools.VsCode.VsCode';
}

public async getExpectedHash(): Promise<string> {
Expand All @@ -69,30 +45,8 @@ export class PlatformData {
return analysis_engine_osx_x64_sha512;
}
if (this.platform.isLinux && this.platform.is64bit) {
const linuxFlavor = await this.getLinuxFlavor();
// tslint:disable-next-line:switch-default
switch (linuxFlavor) {
case 'centos': return analysis_engine_centos_x64_sha512;
case 'debian': return analysis_engine_debian_x64_sha512;
case 'fedora': return analysis_engine_fedora_x64_sha512;
case 'ol': return analysis_engine_ol_x64_sha512;
case 'opensuse': return analysis_engine_opensuse_x64_sha512;
case 'rhel': return analysis_engine_rhel_x64_sha512;
case 'ubuntu': return analysis_engine_ubuntu_x64_sha512;
}
return analysis_engine_linux_x64_sha512;
}
throw new Error('Unknown platform.');
}

private async getLinuxFlavor(): Promise<string> {
const verFile = '/etc/os-release';
const data = await this.fs.readFile(verFile);
if (data) {
const res = /ID=(.*)/.exec(data);
if (res && res.length > 1) {
return res[1];
}
}
return '';
}
}
6 changes: 4 additions & 2 deletions src/client/common/configSettings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ export const IS_WINDOWS = /^win/.test(process.platform);
// tslint:disable-next-line:completed-docs
export class PythonSettings extends EventEmitter implements IPythonSettings {
private static pythonSettings: Map<string, PythonSettings> = new Map<string, PythonSettings>();
public pythiaEnabled = true;
public intelliCodeEnabled = true;
public downloadCodeAnalysis = true;
public jediEnabled = true;
public jediPath = '';
public jediMemoryLimit = 1024;
Expand Down Expand Up @@ -115,6 +116,7 @@ export class PythonSettings extends EventEmitter implements IPythonSettings {
this.venvPath = systemVariables.resolveAny(pythonSettings.get<string>('venvPath'))!;
this.venvFolders = systemVariables.resolveAny(pythonSettings.get<string[]>('venvFolders'))!;

this.downloadCodeAnalysis = systemVariables.resolveAny(pythonSettings.get<boolean>('downloadCodeAnalysis', true))!;
this.jediEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('jediEnabled', true))!;
if (this.jediEnabled) {
// tslint:disable-next-line:no-backbone-get-set-outside-model no-non-null-assertion
Expand All @@ -126,7 +128,7 @@ export class PythonSettings extends EventEmitter implements IPythonSettings {
}
this.jediMemoryLimit = pythonSettings.get<number>('jediMemoryLimit')!;
} else {
this.pythiaEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('pythiaEnabled', true))!;
this.intelliCodeEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('intelliCodeEnabled', true))!;
}

// tslint:disable-next-line:no-backbone-get-set-outside-model no-non-null-assertion
Expand Down
3 changes: 2 additions & 1 deletion src/client/common/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ export interface IPythonSettings {
readonly pythonPath: string;
readonly venvPath: string;
readonly venvFolders: string[];
readonly pythiaEnabled: boolean;
readonly intelliCodeEnabled: boolean;
readonly downloadCodeAnalysis: boolean;
readonly jediEnabled: boolean;
readonly jediPath: string;
readonly jediMemoryLimit: number;
Expand Down

0 comments on commit ca657e0

Please sign in to comment.