diff --git a/extensions/notebook/src/common/utils.ts b/extensions/notebook/src/common/utils.ts index 3acf6e56de..8ee8345671 100644 --- a/extensions/notebook/src/common/utils.ts +++ b/extensions/notebook/src/common/utils.ts @@ -139,6 +139,46 @@ export function getOSPlatformId(): string { return platformId; } +/** + * Compares two version strings to see which is greater. + * @param first First version string to compare. + * @param second Second version string to compare. + * @returns 1 if the first version is greater, -1 if it's less, and 0 otherwise. + */ +export function comparePackageVersions(first: string, second: string): number { + let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr)); + let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr)); + + // If versions have different lengths, then append zeroes to the shorter one + if (firstVersion.length > secondVersion.length) { + let diff = firstVersion.length - secondVersion.length; + secondVersion = secondVersion.concat(new Array(diff).fill(0)); + } else if (secondVersion.length > firstVersion.length) { + let diff = secondVersion.length - firstVersion.length; + firstVersion = firstVersion.concat(new Array(diff).fill(0)); + } + + for (let i = 0; i < firstVersion.length; ++i) { + if (firstVersion[i] > secondVersion[i]) { + return 1; + } else if (firstVersion[i] < secondVersion[i]) { + return -1; + } + } + return 0; +} + +export function sortPackageVersions(versions: string[], ascending: boolean = true) { + return versions.sort((first, second) => { + let compareResult = comparePackageVersions(first, second); + if (ascending) { + return compareResult; + } else { + return compareResult * -1; + } + }); +} + // PRIVATE HELPERS ///////////////////////////////////////////////////////// function outputDataChunk(data: string | Buffer, outputChannel: vscode.OutputChannel, header: string): void { data.toString().split(/\r?\n/) diff --git a/extensions/notebook/src/dialog/managePackages/addNewPackageTab.ts b/extensions/notebook/src/dialog/managePackages/addNewPackageTab.ts index 99125b30ee..673052e88a 100644 --- a/extensions/notebook/src/dialog/managePackages/addNewPackageTab.ts +++ b/extensions/notebook/src/dialog/managePackages/addNewPackageTab.ts @@ -209,7 +209,7 @@ export class AddNewPackageTab { let releaseInfo = packagesJson.releases[versionKey]; return Array.isArray(releaseInfo) && releaseInfo.length > 0; }); - versionNums = AddNewPackageTab.sortPackageVersions(versionKeys); + versionNums = utils.sortPackageVersions(versionKeys, false); } if (packagesJson.info && packagesJson.info.summary) { @@ -247,7 +247,7 @@ export class AddNewPackageTab { if (Array.isArray(packages)) { let allVersions = packages.filter(pkg => pkg && pkg.version).map(pkg => pkg.version); let singletonVersions = new Set(allVersions); - let sortedVersions = AddNewPackageTab.sortPackageVersions(Array.from(singletonVersions)); + let sortedVersions = utils.sortPackageVersions(Array.from(singletonVersions), false); return { name: packageName, versions: sortedVersions, @@ -260,32 +260,6 @@ export class AddNewPackageTab { return undefined; } - public static sortPackageVersions(versions: string[]): string[] { - return versions.sort((first, second) => { - // sort in descending order - let firstVersion = first.split('.').map(numStr => Number.parseInt(numStr)); - let secondVersion = second.split('.').map(numStr => Number.parseInt(numStr)); - - // If versions have different lengths, then append zeroes to the shorter one - if (firstVersion.length > secondVersion.length) { - let diff = firstVersion.length - secondVersion.length; - secondVersion = secondVersion.concat(new Array(diff).fill(0)); - } else if (secondVersion.length > firstVersion.length) { - let diff = secondVersion.length - firstVersion.length; - firstVersion = firstVersion.concat(new Array(diff).fill(0)); - } - - for (let i = 0; i < firstVersion.length; ++i) { - if (firstVersion[i] > secondVersion[i]) { - return -1; - } else if (firstVersion[i] < secondVersion[i]) { - return 1; - } - } - return 0; - }); - } - private async doPackageInstall(): Promise { let packageName = this.newPackagesName.value; let packageVersion = this.newPackagesVersions.value as string; @@ -305,9 +279,9 @@ export class AddNewPackageTab { operation: op => { let installPromise: Promise; if (this.dialog.currentPkgType === PythonPkgType.Anaconda) { - installPromise = this.jupyterInstallation.installCondaPackage(packageName, packageVersion); + installPromise = this.jupyterInstallation.installCondaPackages([{ name: packageName, version: packageVersion }], false); } else { - installPromise = this.jupyterInstallation.installPipPackage(packageName, packageVersion); + installPromise = this.jupyterInstallation.installPipPackages([{ name: packageName, version: packageVersion }], false); } installPromise .then(async () => { @@ -334,4 +308,4 @@ export class AddNewPackageTab { } }); } -} \ No newline at end of file +} diff --git a/extensions/notebook/src/integrationTest/notebookIntegration.test.ts b/extensions/notebook/src/integrationTest/notebookIntegration.test.ts index 8b43bb67ba..1dbdd6ccdf 100644 --- a/extensions/notebook/src/integrationTest/notebookIntegration.test.ts +++ b/extensions/notebook/src/integrationTest/notebookIntegration.test.ts @@ -12,8 +12,7 @@ import 'mocha'; import { JupyterController } from '../jupyter/jupyterController'; import { JupyterServerInstallation, PythonPkgDetails } from '../jupyter/jupyterServerInstallation'; import { pythonBundleVersion } from '../common/constants'; -import { executeStreamedCommand } from '../common/utils'; -import { AddNewPackageTab } from '../dialog/managePackages/addNewPackageTab'; +import { executeStreamedCommand, sortPackageVersions } from '../common/utils'; describe('Notebook Extension Python Installation', function () { this.timeout(600000); @@ -90,7 +89,7 @@ describe('Notebook Extension Python Installation', function () { let testPkgVersion = '0.24.2'; let expectedPkg: PythonPkgDetails = { name: testPkg, version: testPkgVersion }; - await install.installPipPackage(testPkg, testPkgVersion); + await install.installPipPackages([{ name: testPkg, version: testPkgVersion}], false); let packages = await install.getInstalledPipPackages(); should(packages).containEql(expectedPkg); @@ -98,7 +97,7 @@ describe('Notebook Extension Python Installation', function () { packages = await install.getInstalledPipPackages(); should(packages).not.containEql(expectedPkg); - await install.installPipPackage(testPkg, testPkgVersion); + await install.installPipPackages([{ name: testPkg, version: testPkgVersion}], false); packages = await install.getInstalledPipPackages(); should(packages).containEql(expectedPkg); }); @@ -112,16 +111,23 @@ describe('Notebook Extension Python Installation', function () { should(install.getInstalledCondaPackages()).be.rejected(); - should(install.installCondaPackage('pandas', '0.24.2')).be.rejected(); + should(install.installCondaPackages([{ name: 'pandas', version: '0.24.2' }], false)).be.rejected(); should(install.uninstallCondaPackages([{ name: 'pandas', version: '0.24.2' }])).be.rejected(); }); - it('Manage Packages Dialog: New Package Test', async function () { + it('Manage Packages Dialog: Sort Versions Test', async function () { let testVersions = ['1.0.0', '1.1', '0.0.0.9', '0.0.5', '100', '0.3', '3']; - let expectedVerions = ['100', '3', '1.1', '1.0.0', '0.3', '0.0.5', '0.0.0.9']; + let ascendingVersions = ['0.0.0.9', '0.0.5', '0.3', '1.0.0', '1.1', '3', '100']; + let descendingVersions = ['100', '3', '1.1', '1.0.0', '0.3', '0.0.5', '0.0.0.9']; - let actualVersions = AddNewPackageTab.sortPackageVersions(testVersions); - should(actualVersions).be.deepEqual(expectedVerions); + let actualVersions = sortPackageVersions(testVersions); + should(actualVersions).be.deepEqual(ascendingVersions); + + actualVersions = sortPackageVersions(testVersions, true); + should(actualVersions).be.deepEqual(ascendingVersions); + + actualVersions = sortPackageVersions(testVersions, false); + should(actualVersions).be.deepEqual(descendingVersions); }); }); diff --git a/extensions/notebook/src/jupyter/jupyterServerInstallation.ts b/extensions/notebook/src/jupyter/jupyterServerInstallation.ts index 27bec81d61..64c10b5cba 100644 --- a/extensions/notebook/src/jupyter/jupyterServerInstallation.ts +++ b/extensions/notebook/src/jupyter/jupyterServerInstallation.ts @@ -17,6 +17,8 @@ import * as utils from '../common/utils'; import { OutputChannel, ConfigurationTarget, window } from 'vscode'; import { Deferred } from '../common/promise'; import { ConfigurePythonDialog } from '../dialog/configurePythonDialog'; +import { IPrompter, IQuestion, QuestionTypes } from '../prompts/question'; +import CodeAdapter from '../prompts/adapter'; const localize = nls.loadMessageBundle(); const msgInstallPkgProgress = localize('msgInstallPkgProgress', "Notebook dependencies installation is in progress"); @@ -49,25 +51,55 @@ export class JupyterServerInstallation { private _usingExistingPython: boolean; private _usingConda: boolean; - // Allows dependencies to be installed even if an existing installation is already present - private _forceInstall: boolean; private _installInProgress: boolean; public static readonly DefaultPythonLocation = path.join(utils.getUserHome(), 'azuredatastudio-python'); + private _prompter: IPrompter; + + private readonly _commonPackages: PythonPkgDetails[] = [ + { + name: 'jupyter', + version: '1.0.0' + }, { + name: 'pandas', + version: '0.24.2' + }, { + name: 'sparkmagic', + version: '0.12.9' + } + ]; + + private readonly _commonPipPackages: PythonPkgDetails[] = [ + { + name: 'prose-codeaccelerator', + version: '1.3.0' + }, { + name: 'powershell-kernel', + version: '0.1.0' + } + ]; + + private readonly _expectedPythonPackages = this._commonPackages.concat(this._commonPipPackages); + + private readonly _expectedCondaPackages = this._commonPackages.concat([{ name: 'pykerberos', version: '1.2.1' }]); + + private readonly _expectedCondaPipPackages = this._commonPipPackages; + constructor(extensionPath: string, outputChannel: OutputChannel, apiWrapper: ApiWrapper, pythonInstallationPath?: string) { this.extensionPath = extensionPath; this.outputChannel = outputChannel; this.apiWrapper = apiWrapper; this._pythonInstallationPath = pythonInstallationPath || JupyterServerInstallation.getPythonInstallPath(this.apiWrapper); - this._forceInstall = false; this._usingConda = false; this._installInProgress = false; this._usingExistingPython = JupyterServerInstallation.getExistingPythonSetting(this.apiWrapper); + + this._prompter = new CodeAdapter(); } - private async installDependencies(backgroundOperation: azdata.BackgroundOperation): Promise { - if (!(await utils.exists(this._pythonExecutable)) || this._forceInstall || this._usingExistingPython) { + private async installDependencies(backgroundOperation: azdata.BackgroundOperation, forceInstall: boolean): Promise { + if (!(await utils.exists(this._pythonExecutable)) || forceInstall || this._usingExistingPython) { window.showInformationMessage(msgInstallPkgStart); this.outputChannel.show(true); @@ -78,9 +110,9 @@ export class JupyterServerInstallation { await this.installPythonPackage(backgroundOperation); if (this._usingConda) { - await this.installCondaDependencies(); + await this.upgradeCondaPackages(false, forceInstall); } else if (this._usingExistingPython) { - await this.installPipDependencies(); + await this.upgradePythonPackages(false, forceInstall); } else { await this.installOfflinePipDependencies(); } @@ -301,7 +333,6 @@ export class JupyterServerInstallation { } this._installInProgress = true; - this._forceInstall = forceInstall; if (installSettings) { this._pythonInstallationPath = installSettings.installPath; this._usingExistingPython = installSettings.existingPython; @@ -315,13 +346,13 @@ export class JupyterServerInstallation { await this.configurePackagePaths(); }; let installReady = new Deferred(); - if (!(await utils.exists(this._pythonExecutable)) || this._forceInstall || this._usingExistingPython) { + if (!(await utils.exists(this._pythonExecutable)) || forceInstall || this._usingExistingPython) { this.apiWrapper.startBackgroundOperation({ displayName: msgTaskName, description: msgTaskName, isCancelable: false, operation: op => { - this.installDependencies(op) + this.installDependencies(op, forceInstall) .then(async () => { await updateConfig(); installReady.resolve(); @@ -356,6 +387,106 @@ export class JupyterServerInstallation { } } + /** + * Prompts user to upgrade certain python packages if they're below the minimum expected version. + */ + public promptForPackageUpgrade(): Promise { + if (this._usingConda) { + return this.upgradeCondaPackages(true, false); + } else { + return this.upgradePythonPackages(true, false); + } + } + + private async upgradePythonPackages(promptForUpgrade: boolean, forceInstall: boolean): Promise { + let installedPackages = await this.getInstalledPipPackages(); + let pkgVersionMap = new Map(); + installedPackages.forEach(pkg => pkgVersionMap.set(pkg.name, pkg.version)); + + let packagesToInstall: PythonPkgDetails[]; + if (forceInstall) { + packagesToInstall = this._expectedPythonPackages; + } else { + packagesToInstall = []; + this._expectedPythonPackages.forEach(expectedPkg => { + let installedPkgVersion = pkgVersionMap.get(expectedPkg.name); + if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) { + packagesToInstall.push(expectedPkg); + } + }); + } + + if (packagesToInstall.length > 0) { + let doUpgrade: boolean; + if (promptForUpgrade) { + doUpgrade = await this._prompter.promptSingle({ + type: QuestionTypes.confirm, + message: localize('confirmPipUpgrade', "Some installed pip packages need to be upgraded. Would you like to upgrade them now?"), + default: true + }); + } else { + doUpgrade = true; + } + if (doUpgrade) { + await this.installPipPackages(packagesToInstall, true); + } + } + } + + private async upgradeCondaPackages(promptForUpgrade: boolean, forceInstall: boolean): Promise { + let condaPackagesToInstall: PythonPkgDetails[] = []; + let pipPackagesToInstall: PythonPkgDetails[] = []; + + if (forceInstall) { + condaPackagesToInstall = this._expectedCondaPackages; + pipPackagesToInstall = this._expectedCondaPipPackages; + } else { + condaPackagesToInstall = []; + pipPackagesToInstall = []; + + // Conda packages + let installedCondaPackages = await this.getInstalledCondaPackages(); + let condaVersionMap = new Map(); + installedCondaPackages.forEach(pkg => condaVersionMap.set(pkg.name, pkg.version)); + + this._expectedCondaPackages.forEach(expectedPkg => { + let installedPkgVersion = condaVersionMap.get(expectedPkg.name); + if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) { + condaPackagesToInstall.push(expectedPkg); + } + }); + + // Pip packages + let installedPipPackages = await this.getInstalledPipPackages(); + let pipVersionMap = new Map(); + installedPipPackages.forEach(pkg => pipVersionMap.set(pkg.name, pkg.version)); + + this._expectedCondaPipPackages.forEach(expectedPkg => { + let installedPkgVersion = pipVersionMap.get(expectedPkg.name); + if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, expectedPkg.version) < 0) { + pipPackagesToInstall.push(expectedPkg); + } + }); + } + + if (condaPackagesToInstall.length > 0 || pipPackagesToInstall.length > 0) { + let doUpgrade: boolean; + if (promptForUpgrade) { + doUpgrade = await this._prompter.promptSingle({ + type: QuestionTypes.confirm, + message: localize('confirmCondaUpgrade', "Some installed conda and pip packages need to be upgraded. Would you like to upgrade them now?"), + default: true + }); + } else { + doUpgrade = true; + } + if (doUpgrade) { + await this.installCondaPackages(condaPackagesToInstall, true); + await this.installPipPackages(pipPackagesToInstall, true); + } + } + } + public async getInstalledPipPackages(): Promise { let cmd = `"${this.pythonExecutable}" -m pip list --format=json`; let packagesInfo = await this.executeBufferedCommand(cmd); @@ -367,15 +498,21 @@ export class JupyterServerInstallation { return packagesResult; } - public installPipPackage(packageName: string, version: string): Promise { + public installPipPackages(packages: PythonPkgDetails[], useMinVersion: boolean): Promise { + if (!packages || packages.length === 0) { + return Promise.resolve(); + } + + let versionSpecifier = useMinVersion ? '>=' : '=='; + let packagesStr = packages.map(pkg => `"${pkg.name}${versionSpecifier}${pkg.version}"`).join(' '); // Force reinstall in case some dependencies are split across multiple locations let cmdOptions = this._usingExistingPython ? '--user --force-reinstall' : '--force-reinstall'; - let cmd = `"${this.pythonExecutable}" -m pip install ${cmdOptions} ${packageName}==${version}`; + let cmd = `"${this.pythonExecutable}" -m pip install ${cmdOptions} ${packagesStr} --extra-index-url https://prose-python-packages.azurewebsites.net`; return this.executeStreamedCommand(cmd); } public uninstallPipPackages(packages: PythonPkgDetails[]): Promise { - let packagesStr = packages.map(pkg => `${pkg.name}==${pkg.version}`).join(' '); + let packagesStr = packages.map(pkg => `"${pkg.name}==${pkg.version}"`).join(' '); let cmd = `"${this.pythonExecutable}" -m pip uninstall -y ${packagesStr}`; return this.executeStreamedCommand(cmd); } @@ -396,15 +533,21 @@ export class JupyterServerInstallation { return []; } - public installCondaPackage(packageName: string, version: string): Promise { + public installCondaPackages(packages: PythonPkgDetails[], useMinVersion: boolean): Promise { + if (!packages || packages.length === 0) { + return Promise.resolve(); + } + + let versionSpecifier = useMinVersion ? '>=' : '=='; + let packagesStr = packages.map(pkg => `"${pkg.name}${versionSpecifier}${pkg.version}"`).join(' '); let condaExe = this.getCondaExePath(); - let cmd = `"${condaExe}" install -y ${packageName}==${version}`; + let cmd = `"${condaExe}" install -y --force-reinstall ${packagesStr}`; return this.executeStreamedCommand(cmd); } public uninstallCondaPackages(packages: PythonPkgDetails[]): Promise { let condaExe = this.getCondaExePath(); - let packagesStr = packages.map(pkg => `${pkg.name}==${pkg.version}`).join(' '); + let packagesStr = packages.map(pkg => `"${pkg.name}==${pkg.version}"`).join(' '); let cmd = `"${condaExe}" uninstall -y ${packagesStr}`; return this.executeStreamedCommand(cmd); } @@ -435,42 +578,6 @@ export class JupyterServerInstallation { } } - private async installPipDependencies(): Promise { - this.outputChannel.show(true); - this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks...")); - - let packages = 'jupyter>=1.0.0 pandas>=0.24.2 sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 powershell_kernel>=0.1.0'; - let cmdOptions = this._usingExistingPython ? '--user' : ''; - if (this._forceInstall) { - cmdOptions = `${cmdOptions} --force-reinstall`; - } - let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} ${packages} --extra-index-url https://prose-python-packages.azurewebsites.net`; - await this.executeStreamedCommand(installCommand); - - this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete.")); - } - - private async installCondaDependencies(): Promise { - this.outputChannel.show(true); - this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks...")); - - let installCommand = `"${this.getCondaExePath()}" install -y jupyter>=1.0.0 pandas>=0.24.2`; - if (process.platform !== constants.winPlatform) { - installCommand = `${installCommand} pykerberos>=1.2.1`; - } - await this.executeStreamedCommand(installCommand); - - let pipPackages = 'sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 powershell_kernel>=0.1.0'; - let cmdOptions = this._usingExistingPython ? '--user' : ''; - if (this._forceInstall) { - cmdOptions = `${cmdOptions} --force-reinstall`; - } - installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} ${pipPackages} --extra-index-url https://prose-python-packages.azurewebsites.net`; - await this.executeStreamedCommand(installCommand); - - this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete.")); - } - private async executeStreamedCommand(command: string): Promise { await utils.executeStreamedCommand(command, { env: this.execOptions.env }, this.outputChannel); } diff --git a/extensions/notebook/src/jupyter/jupyterServerManager.ts b/extensions/notebook/src/jupyter/jupyterServerManager.ts index 09f5b04473..4f6907b3e0 100644 --- a/extensions/notebook/src/jupyter/jupyterServerManager.ts +++ b/extensions/notebook/src/jupyter/jupyterServerManager.ts @@ -105,6 +105,7 @@ export class LocalJupyterServerManager implements nb.ServerManager, vscode.Dispo private async doStartServer(): Promise { // We can't find or create servers until the installation is complete let installation = this.options.jupyterInstallation; await installation.promptForPythonInstall(); + await installation.promptForPackageUpgrade(); this.apiWrapper.setCommandContext(CommandContext.NotebookPythonInstalled, true); // Calculate the path to use as the notebook-dir for Jupyter based on the path of the uri of the