ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-01-14 01:25:37 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add prompt for user to upgrade python packages when starting a notebook (#7574)

Browse Source
This commit is contained in:
Cory Rivera
2019-10-08 17:42:10 -07:00
committed by GitHub
parent d37105ada2
commit 3a01f960a7
5 changed files with 220 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
extensions/notebook/src/common/utils.ts
View File

@@ -139,6 +139,46 @@ export function getOSPlatformId(): string {
return platformId;
}
/**
* Compares two version strings to see which is greater.
* @param first First version string to compare.
* @param second Second version string to compare.
* @returns 1 if the first version is greater, -1 if it's less, and 0 otherwise.
*/
export function comparePackageVersions(first: string, second: string): number {
let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr));
let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr));
// If versions have different lengths, then append zeroes to the shorter one
if (firstVersion.length > secondVersion.length) {
let diff = firstVersion.length - secondVersion.length;
secondVersion = secondVersion.concat(new Array(diff).fill(0));
} else if (secondVersion.length > firstVersion.length) {
let diff = secondVersion.length - firstVersion.length;
firstVersion = firstVersion.concat(new Array(diff).fill(0));
}
for (let i = 0; i < firstVersion.length; ++i) {
if (firstVersion[i] > secondVersion[i]) {
return 1;
} else if (firstVersion[i] < secondVersion[i]) {
return -1;
}
}
return 0;
}
export function sortPackageVersions(versions: string[], ascending: boolean = true) {
return versions.sort((first, second) => {
let compareResult = comparePackageVersions(first, second);
if (ascending) {
return compareResult;
} else {
return compareResult * -1;
}
});
}
// PRIVATE HELPERS /////////////////////////////////////////////////////////
function outputDataChunk(data: string | Buffer, outputChannel: vscode.OutputChannel, header: string): void {
data.toString().split(/\r?\n/)

36
extensions/notebook/src/dialog/managePackages/addNewPackageTab.ts
View File

@@ -209,7 +209,7 @@ export class AddNewPackageTab {
let releaseInfo = packagesJson.releases[versionKey];
return Array.isArray(releaseInfo) && releaseInfo.length > 0;
});
versionNums = AddNewPackageTab.sortPackageVersions(versionKeys);
versionNums = utils.sortPackageVersions(versionKeys, false);
}
if (packagesJson.info && packagesJson.info.summary) {
@@ -247,7 +247,7 @@ export class AddNewPackageTab {
if (Array.isArray(packages)) {
let allVersions = packages.filter(pkg => pkg && pkg.version).map(pkg => pkg.version);
let singletonVersions = new Set<string>(allVersions);
let sortedVersions = AddNewPackageTab.sortPackageVersions(Array.from(singletonVersions));
let sortedVersions = utils.sortPackageVersions(Array.from(singletonVersions), false);
return {
name: packageName,
versions: sortedVersions,
@@ -260,32 +260,6 @@ export class AddNewPackageTab {
return undefined;
}
public static sortPackageVersions(versions: string[]): string[] {
return versions.sort((first, second) => {
// sort in descending order
let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr));
let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr));
// If versions have different lengths, then append zeroes to the shorter one
if (firstVersion.length > secondVersion.length) {
let diff = firstVersion.length - secondVersion.length;
secondVersion = secondVersion.concat(new Array(diff).fill(0));
} else if (secondVersion.length > firstVersion.length) {
let diff = secondVersion.length - firstVersion.length;
firstVersion = firstVersion.concat(new Array(diff).fill(0));
}
for (let i = 0; i < firstVersion.length; ++i) {
if (firstVersion[i] > secondVersion[i]) {
return -1;
} else if (firstVersion[i] < secondVersion[i]) {
return 1;
}
}
return 0;
});
}
private async doPackageInstall(): Promise<void> {
let packageName = this.newPackagesName.value;
let packageVersion = this.newPackagesVersions.value as string;
@@ -305,9 +279,9 @@ export class AddNewPackageTab {
operation: op => {
let installPromise: Promise<void>;
if (this.dialog.currentPkgType === PythonPkgType.Anaconda) {
installPromise = this.jupyterInstallation.installCondaPackage(packageName, packageVersion);
installPromise = this.jupyterInstallation.installCondaPackages([{ name: packageName, version: packageVersion }], false);
} else {
installPromise = this.jupyterInstallation.installPipPackage(packageName, packageVersion);
installPromise = this.jupyterInstallation.installPipPackages([{ name: packageName, version: packageVersion }], false);
}
installPromise
.then(async () => {
@@ -334,4 +308,4 @@ export class AddNewPackageTab {
}
});
}
}
}

24
extensions/notebook/src/integrationTest/notebookIntegration.test.ts
View File

@@ -12,8 +12,7 @@ import 'mocha';
import { JupyterController } from '../jupyter/jupyterController';
import { JupyterServerInstallation, PythonPkgDetails } from '../jupyter/jupyterServerInstallation';
import { pythonBundleVersion } from '../common/constants';
import { executeStreamedCommand } from '../common/utils';
import { AddNewPackageTab } from '../dialog/managePackages/addNewPackageTab';
import { executeStreamedCommand, sortPackageVersions } from '../common/utils';
describe('Notebook Extension Python Installation', function () {
this.timeout(600000);
@@ -90,7 +89,7 @@ describe('Notebook Extension Python Installation', function () {
let testPkgVersion = '0.24.2';
let expectedPkg: PythonPkgDetails = { name: testPkg, version: testPkgVersion };
await install.installPipPackage(testPkg, testPkgVersion);
await install.installPipPackages([{ name: testPkg, version: testPkgVersion}], false);
let packages = await install.getInstalledPipPackages();
should(packages).containEql(expectedPkg);
@@ -98,7 +97,7 @@ describe('Notebook Extension Python Installation', function () {
packages = await install.getInstalledPipPackages();
should(packages).not.containEql(expectedPkg);
await install.installPipPackage(testPkg, testPkgVersion);
await install.installPipPackages([{ name: testPkg, version: testPkgVersion}], false);
packages = await install.getInstalledPipPackages();
should(packages).containEql(expectedPkg);
});
@@ -112,16 +111,23 @@ describe('Notebook Extension Python Installation', function () {
should(install.getInstalledCondaPackages()).be.rejected();
should(install.installCondaPackage('pandas', '0.24.2')).be.rejected();
should(install.installCondaPackages([{ name: 'pandas', version: '0.24.2' }], false)).be.rejected();
should(install.uninstallCondaPackages([{ name: 'pandas', version: '0.24.2' }])).be.rejected();
});
it('Manage Packages Dialog: New Package Test', async function () {
it('Manage Packages Dialog: Sort Versions Test', async function () {
let testVersions = ['1.0.0', '1.1', '0.0.0.9', '0.0.5', '100', '0.3', '3'];
let expectedVerions = ['100', '3', '1.1', '1.0.0', '0.3', '0.0.5', '0.0.0.9'];
let ascendingVersions = ['0.0.0.9', '0.0.5', '0.3', '1.0.0', '1.1', '3', '100'];
let descendingVersions = ['100', '3', '1.1', '1.0.0', '0.3', '0.0.5', '0.0.0.9'];
let actualVersions = AddNewPackageTab.sortPackageVersions(testVersions);
should(actualVersions).be.deepEqual(expectedVerions);
let actualVersions = sortPackageVersions(testVersions);
should(actualVersions).be.deepEqual(ascendingVersions);
actualVersions = sortPackageVersions(testVersions, true);
should(actualVersions).be.deepEqual(ascendingVersions);
actualVersions = sortPackageVersions(testVersions, false);
should(actualVersions).be.deepEqual(descendingVersions);
});
});

211
extensions/notebook/src/jupyter/jupyterServerInstallation.ts
View File

@@ -17,6 +17,8 @@ import * as utils from '../common/utils';
import { OutputChannel, ConfigurationTarget, window } from 'vscode';
import { Deferred } from '../common/promise';
import { ConfigurePythonDialog } from '../dialog/configurePythonDialog';
import { IPrompter, IQuestion, QuestionTypes } from '../prompts/question';
import CodeAdapter from '../prompts/adapter';
const localize = nls.loadMessageBundle();
const msgInstallPkgProgress = localize('msgInstallPkgProgress', "Notebook dependencies installation is in progress");
@@ -49,25 +51,55 @@ export class JupyterServerInstallation {
private _usingExistingPython: boolean;
private _usingConda: boolean;
// Allows dependencies to be installed even if an existing installation is already present
private _forceInstall: boolean;
private _installInProgress: boolean;
public static readonly DefaultPythonLocation = path.join(utils.getUserHome(), 'azuredatastudio-python');
private _prompter: IPrompter;
private readonly _commonPackages: PythonPkgDetails[] = [
{
name: 'jupyter',
version: '1.0.0'
}, {
name: 'pandas',
version: '0.24.2'
}, {
name: 'sparkmagic',
version: '0.12.9'
}
];
private readonly _commonPipPackages: PythonPkgDetails[] = [
{
name: 'prose-codeaccelerator',
version: '1.3.0'
}, {
name: 'powershell-kernel',
version: '0.1.0'
}
];
private readonly _expectedPythonPackages = this._commonPackages.concat(this._commonPipPackages);
private readonly _expectedCondaPackages = this._commonPackages.concat([{ name: 'pykerberos', version: '1.2.1' }]);
private readonly _expectedCondaPipPackages = this._commonPipPackages;
constructor(extensionPath: string, outputChannel: OutputChannel, apiWrapper: ApiWrapper, pythonInstallationPath?: string) {
this.extensionPath = extensionPath;
this.outputChannel = outputChannel;
this.apiWrapper = apiWrapper;
this._pythonInstallationPath = pythonInstallationPath || JupyterServerInstallation.getPythonInstallPath(this.apiWrapper);
this._forceInstall = false;
this._usingConda = false;
this._installInProgress = false;
this._usingExistingPython = JupyterServerInstallation.getExistingPythonSetting(this.apiWrapper);
this._prompter = new CodeAdapter();
}
private async installDependencies(backgroundOperation: azdata.BackgroundOperation): Promise<void> {
if (!(await utils.exists(this._pythonExecutable)) || this._forceInstall || this._usingExistingPython) {
private async installDependencies(backgroundOperation: azdata.BackgroundOperation, forceInstall: boolean): Promise<void> {
if (!(await utils.exists(this._pythonExecutable)) || forceInstall || this._usingExistingPython) {
window.showInformationMessage(msgInstallPkgStart);
this.outputChannel.show(true);
@@ -78,9 +110,9 @@ export class JupyterServerInstallation {
await this.installPythonPackage(backgroundOperation);
if (this._usingConda) {
await this.installCondaDependencies();
await this.upgradeCondaPackages(false, forceInstall);
} else if (this._usingExistingPython) {
await this.installPipDependencies();
await this.upgradePythonPackages(false, forceInstall);
} else {
await this.installOfflinePipDependencies();
}
@@ -301,7 +333,6 @@ export class JupyterServerInstallation {
}
this._installInProgress = true;
this._forceInstall = forceInstall;
if (installSettings) {
this._pythonInstallationPath = installSettings.installPath;
this._usingExistingPython = installSettings.existingPython;
@@ -315,13 +346,13 @@ export class JupyterServerInstallation {
await this.configurePackagePaths();
};
let installReady = new Deferred<void>();
if (!(await utils.exists(this._pythonExecutable)) || this._forceInstall || this._usingExistingPython) {
if (!(await utils.exists(this._pythonExecutable)) || forceInstall || this._usingExistingPython) {
this.apiWrapper.startBackgroundOperation({
displayName: msgTaskName,
description: msgTaskName,
isCancelable: false,
operation: op => {
this.installDependencies(op)
this.installDependencies(op, forceInstall)
.then(async () => {
await updateConfig();
installReady.resolve();
@@ -356,6 +387,106 @@ export class JupyterServerInstallation {
}
}
/**
* Prompts user to upgrade certain python packages if they're below the minimum expected version.
*/
public promptForPackageUpgrade(): Promise<void> {
if (this._usingConda) {
return this.upgradeCondaPackages(true, false);
} else {
return this.upgradePythonPackages(true, false);
}
}
private async upgradePythonPackages(promptForUpgrade: boolean, forceInstall: boolean): Promise<void> {
let installedPackages = await this.getInstalledPipPackages();
let pkgVersionMap = new Map<string, string>();
installedPackages.forEach(pkg => pkgVersionMap.set(pkg.name, pkg.version));
let packagesToInstall: PythonPkgDetails[];
if (forceInstall) {
packagesToInstall = this._expectedPythonPackages;
} else {
packagesToInstall = [];
this._expectedPythonPackages.forEach(expectedPkg => {
let installedPkgVersion = pkgVersionMap.get(expectedPkg.name);
if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) {
packagesToInstall.push(expectedPkg);
}
});
}
if (packagesToInstall.length > 0) {
let doUpgrade: boolean;
if (promptForUpgrade) {
doUpgrade = await this._prompter.promptSingle<boolean>(<IQuestion>{
type: QuestionTypes.confirm,
message: localize('confirmPipUpgrade', "Some installed pip packages need to be upgraded. Would you like to upgrade them now?"),
default: true
});
} else {
doUpgrade = true;
}
if (doUpgrade) {
await this.installPipPackages(packagesToInstall, true);
}
}
}
private async upgradeCondaPackages(promptForUpgrade: boolean, forceInstall: boolean): Promise<void> {
let condaPackagesToInstall: PythonPkgDetails[] = [];
let pipPackagesToInstall: PythonPkgDetails[] = [];
if (forceInstall) {
condaPackagesToInstall = this._expectedCondaPackages;
pipPackagesToInstall = this._expectedCondaPipPackages;
} else {
condaPackagesToInstall = [];
pipPackagesToInstall = [];
// Conda packages
let installedCondaPackages = await this.getInstalledCondaPackages();
let condaVersionMap = new Map<string, string>();
installedCondaPackages.forEach(pkg => condaVersionMap.set(pkg.name, pkg.version));
this._expectedCondaPackages.forEach(expectedPkg => {
let installedPkgVersion = condaVersionMap.get(expectedPkg.name);
if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) {
condaPackagesToInstall.push(expectedPkg);
}
});
// Pip packages
let installedPipPackages = await this.getInstalledPipPackages();
let pipVersionMap = new Map<string, string>();
installedPipPackages.forEach(pkg => pipVersionMap.set(pkg.name, pkg.version));
this._expectedCondaPipPackages.forEach(expectedPkg => {
let installedPkgVersion = pipVersionMap.get(expectedPkg.name);
if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) {
pipPackagesToInstall.push(expectedPkg);
}
});
}
if (condaPackagesToInstall.length > 0 || pipPackagesToInstall.length > 0) {
let doUpgrade: boolean;
if (promptForUpgrade) {
doUpgrade = await this._prompter.promptSingle<boolean>(<IQuestion>{
type: QuestionTypes.confirm,
message: localize('confirmCondaUpgrade', "Some installed conda and pip packages need to be upgraded. Would you like to upgrade them now?"),
default: true
});
} else {
doUpgrade = true;
}
if (doUpgrade) {
await this.installCondaPackages(condaPackagesToInstall, true);
await this.installPipPackages(pipPackagesToInstall, true);
}
}
}
public async getInstalledPipPackages(): Promise<PythonPkgDetails[]> {
let cmd = `"${this.pythonExecutable}" -m pip list --format=json`;
let packagesInfo = await this.executeBufferedCommand(cmd);
@@ -367,15 +498,21 @@ export class JupyterServerInstallation {
return packagesResult;
}
public installPipPackage(packageName: string, version: string): Promise<void> {
public installPipPackages(packages: PythonPkgDetails[], useMinVersion: boolean): Promise<void> {
if (!packages || packages.length === 0) {
return Promise.resolve();
}
let versionSpecifier = useMinVersion ? '>=' : '==';
let packagesStr = packages.map(pkg => `"${pkg.name}${versionSpecifier}${pkg.version}"`).join(' ');
// Force reinstall in case some dependencies are split across multiple locations
let cmdOptions = this._usingExistingPython ? '--user --force-reinstall' : '--force-reinstall';
let cmd = `"${this.pythonExecutable}" -m pip install ${cmdOptions} ${packageName}==${version}`;
let cmd = `"${this.pythonExecutable}" -m pip install ${cmdOptions} ${packagesStr} --extra-index-url https://prose-python-packages.azurewebsites.net`;
return this.executeStreamedCommand(cmd);
}
public uninstallPipPackages(packages: PythonPkgDetails[]): Promise<void> {
let packagesStr = packages.map(pkg => `${pkg.name}==${pkg.version}`).join(' ');
let packagesStr = packages.map(pkg => `"${pkg.name}==${pkg.version}"`).join(' ');
let cmd = `"${this.pythonExecutable}" -m pip uninstall -y ${packagesStr}`;
return this.executeStreamedCommand(cmd);
}
@@ -396,15 +533,21 @@ export class JupyterServerInstallation {
return [];
}
public installCondaPackage(packageName: string, version: string): Promise<void> {
public installCondaPackages(packages: PythonPkgDetails[], useMinVersion: boolean): Promise<void> {
if (!packages || packages.length === 0) {
return Promise.resolve();
}
let versionSpecifier = useMinVersion ? '>=' : '==';
let packagesStr = packages.map(pkg => `"${pkg.name}${versionSpecifier}${pkg.version}"`).join(' ');
let condaExe = this.getCondaExePath();
let cmd = `"${condaExe}" install -y ${packageName}==${version}`;
let cmd = `"${condaExe}" install -y --force-reinstall ${packagesStr}`;
return this.executeStreamedCommand(cmd);
}
public uninstallCondaPackages(packages: PythonPkgDetails[]): Promise<void> {
let condaExe = this.getCondaExePath();
let packagesStr = packages.map(pkg => `${pkg.name}==${pkg.version}`).join(' ');
let packagesStr = packages.map(pkg => `"${pkg.name}==${pkg.version}"`).join(' ');
let cmd = `"${condaExe}" uninstall -y ${packagesStr}`;
return this.executeStreamedCommand(cmd);
}
@@ -435,42 +578,6 @@ export class JupyterServerInstallation {
}
}
private async installPipDependencies(): Promise<void> {
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks..."));
let packages = 'jupyter>=1.0.0 pandas>=0.24.2 sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 powershell_kernel>=0.1.0';
let cmdOptions = this._usingExistingPython ? '--user' : '';
if (this._forceInstall) {
cmdOptions = `${cmdOptions} --force-reinstall`;
}
let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} ${packages} --extra-index-url https://prose-python-packages.azurewebsites.net`;
await this.executeStreamedCommand(installCommand);
this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete."));
}
private async installCondaDependencies(): Promise<void> {
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks..."));
let installCommand = `"${this.getCondaExePath()}" install -y jupyter>=1.0.0 pandas>=0.24.2`;
if (process.platform !== constants.winPlatform) {
installCommand = `${installCommand} pykerberos>=1.2.1`;
}
await this.executeStreamedCommand(installCommand);
let pipPackages = 'sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 powershell_kernel>=0.1.0';
let cmdOptions = this._usingExistingPython ? '--user' : '';
if (this._forceInstall) {
cmdOptions = `${cmdOptions} --force-reinstall`;
}
installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} ${pipPackages} --extra-index-url https://prose-python-packages.azurewebsites.net`;
await this.executeStreamedCommand(installCommand);
this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete."));
}
private async executeStreamedCommand(command: string): Promise<void> {
await utils.executeStreamedCommand(command, { env: this.execOptions.env }, this.outputChannel);
}

1
extensions/notebook/src/jupyter/jupyterServerManager.ts
View File

@@ -105,6 +105,7 @@ export class LocalJupyterServerManager implements nb.ServerManager, vscode.Dispo
private async doStartServer(): Promise<IServerInstance> { // We can't find or create servers until the installation is complete
let installation = this.options.jupyterInstallation;
await installation.promptForPythonInstall();
await installation.promptForPackageUpgrade();
this.apiWrapper.setCommandContext(CommandContext.NotebookPythonInstalled, true);
// Calculate the path to use as the notebook-dir for Jupyter based on the path of the uri of the