Update Python to 3.8.8 (#15278)

* update python fwlinks and remove bundle ver

* start fixing path for users with python 36

* prompt user for python version upgrade

* update python path after removing 3.6

* prompt users to upgrade and show pkg warning

* make prompt async

* remove python bundle ver from ML extension

* shutdown python 3.6 before deleting

* check useExistingPython before update prompt

* add dont ask again option

* remove 3.6 after installing 3.8

fix merge conflict

* give option to remove python36

* list user installed pip packages in warning

* create notebook to install pip packages

* update getPythonExePath method and add comments

* clean up code

* add comments

* pr comments

* add comment

* remove option to keep python36

* shutdown active servers before removing python36

* fix error removing old python w/ path change

* update to 3.8.10

* restart sessions for mac/linux

extensions/machine-learning/src/common/constants.ts

@@ -8,7 +8,6 @@ import * as nls from 'vscode-nls';
 const localize = nls.loadMessageBundle();
 
 export const winPlatform = 'win32';
-export const pythonBundleVersion = '0.0.1';
 export const managePackagesCommand = 'jupyter.cmd.managePackages';
 export const pythonLanguageName = 'Python';
 export const rLanguageName = 'R';
@@ -42,7 +41,6 @@ export const pythonEnabledConfigKey = 'enablePython';
 export const rEnabledConfigKey = 'enableR';
 export const registeredModelsTableName = 'registeredModelsTableName';
 export const rPathConfigKey = 'rPath';
-export const adsPythonBundleVersion = '0.0.1';
 
 // TSQL
 //

extensions/machine-learning/src/common/utils.ts

@@ -64,13 +64,6 @@ export function getPythonInstallationLocation(rootFolder: string) {
 	return path.join(rootFolder, 'python');
 }
 
-export function getPythonExePath(rootFolder: string): string {
-	return path.join(
-		getPythonInstallationLocation(rootFolder),
-		constants.pythonBundleVersion,
-		process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
-}
-
 export function getPackageFilePath(rootFolder: string, packageName: string): string {
 	return path.join(
 		rootFolder,
@@ -272,7 +265,6 @@ export function getFileName(filePath: string) {
 export function getDefaultPythonLocation(): string {
 
 	return path.join(getUserHome() || '', 'azuredatastudio-python',
-		constants.adsPythonBundleVersion,
 		getPythonExeName());
 }
 

extensions/notebook/package.json

@@ -38,6 +38,11 @@
           "default": false,
           "description": "%notebook.useExistingPython.description%"
         },
+        "notebook.dontPromptPythonUpdate": {
+          "type": "boolean",
+          "default": false,
+          "description": "%notebook.dontPromptPythonUpdate.description%"
+        },
         "notebook.overrideEditorTheming": {
           "type": "boolean",
           "default": true,

extensions/notebook/package.nls.json

@@ -4,6 +4,7 @@
 	"notebook.configuration.title": "Notebook configuration",
 	"notebook.pythonPath.description": "Local path to python installation used by Notebooks.",
 	"notebook.useExistingPython.description": "Local path to a preexisting python installation used by Notebooks.",
+	"notebook.dontPromptPythonUpdate.description": "Do not show prompt to update Python.",
 	"notebook.overrideEditorTheming.description": "Override editor default settings in the Notebook editor. Settings include background color, current line color and border",
 	"notebook.maxTableRows.description": "Maximum number of rows returned per table in the Notebook editor",
 	"notebook.trustedBooks.description": "Notebooks contained in these books will automatically be trusted.",

extensions/notebook/src/common/constants.ts

@@ -13,10 +13,10 @@ export const extensionOutputChannelName = 'Notebooks';
 export const notebookCommandNew = 'notebook.command.new';
 
 // JUPYTER CONFIG //////////////////////////////////////////////////////////
-export const pythonBundleVersion = '0.0.1';
+export const pythonVersion = '3.8.10';
-export const pythonVersion = '3.6.6';
 export const pythonPathConfigKey = 'pythonPath';
 export const existingPythonConfigKey = 'useExistingPython';
+export const dontPromptPythonUpdate = 'dontPromptPythonUpdate';
 export const notebookConfigKey = 'notebook';
 export const trustedBooksConfigKey = 'trustedBooks';
 export const pinnedBooksConfigKey = 'pinnedNotebooks';
@@ -76,9 +76,9 @@ export enum NavigationProviders {
 export const unsavedBooksContextKey = 'unsavedBooks';
 export const showPinnedBooksContextKey = 'showPinnedbooks';
 
-export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2110625';
+export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2163338';
-export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2128152';
+export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2163337';
-export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2110524';
+export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2163336';
 
 export const notebookLanguages = ['notebook', 'ipynb'];
 

extensions/notebook/src/common/notebookUtils.ts

@@ -19,17 +19,9 @@ export class NotebookUtils {
 
 	constructor() { }
 
-	public async newNotebook(connectionProfile?: azdata.IConnectionProfile): Promise<azdata.nb.NotebookEditor> {
+	public async newNotebook(options?: azdata.nb.NotebookShowOptions): Promise<azdata.nb.NotebookEditor> {
 		const title = this.findNextUntitledEditorName();
 		const untitledUri = vscode.Uri.parse(`untitled:${title}`);
-		const options: azdata.nb.NotebookShowOptions = connectionProfile ? {
-			viewColumn: null,
-			preserveFocus: true,
-			preview: null,
-			providerId: null,
-			connectionProfile: connectionProfile,
-			defaultKernel: null
-		} : null;
 		return azdata.nb.showNotebookDocument(untitledUri, options);
 	}
 

extensions/notebook/src/common/utils.ts

@@ -140,7 +140,7 @@ export function getOSPlatformId(): string {
  * @param second Second version string to compare.
  * @returns 1 if the first version is greater, -1 if it's less, and 0 otherwise.
  */
-export function comparePackageVersions(first: string, second: string): number {
+export function compareVersions(first: string, second: string): number {
 	let firstVersion = first.split('.');
 	let secondVersion = second.split('.');
 
@@ -179,7 +179,7 @@ export function comparePackageVersions(first: string, second: string): number {
 
 export function sortPackageVersions(versions: string[], ascending: boolean = true): string[] {
 	return versions.sort((first, second) => {
-		let compareResult = comparePackageVersions(first, second);
+		let compareResult = compareVersions(first, second);
 		if (ascending) {
 			return compareResult;
 		} else {
@@ -230,7 +230,7 @@ export function isPackageSupported(pythonVersion: string, packageVersionConstrai
 				versionSpecifier = constraint.slice(0, splitIndex);
 				version = constraint.slice(splitIndex).trim();
 			}
-			let versionComparison = comparePackageVersions(pythonVersion, version);
+			let versionComparison = compareVersions(pythonVersion, version);
 			switch (versionSpecifier) {
 				case '>=':
 					supportedVersionFound = versionComparison !== -1;

extensions/notebook/src/dialog/configurePython/configurePythonWizard.ts

@@ -149,7 +149,7 @@ export class ConfigurePythonWizard {
 			}
 
 			if (useExistingPython) {
-				let exePath = JupyterServerInstallation.getPythonExePath(pythonLocation, true);
+				let exePath = JupyterServerInstallation.getPythonExePath(pythonLocation);
 				let pythonExists = await utils.exists(exePath);
 				if (!pythonExists) {
 					this.showErrorMessage(this.PythonNotFoundMsg);

extensions/notebook/src/dialog/configurePython/pickPackagesPage.ts

@@ -110,7 +110,7 @@ export class PickPackagesPage extends BasePage {
 
 	public async onPageEnter(): Promise<void> {
 		this.packageVersionMap.clear();
-		let pythonExe = JupyterServerInstallation.getPythonExePath(this.model.pythonLocation, this.model.useExistingPython);
+		let pythonExe = JupyterServerInstallation.getPythonExePath(this.model.pythonLocation);
 		this.packageVersionRetrieval = this.model.installation.getInstalledPipPackages(pythonExe)
 			.then(installedPackages => {
 				if (installedPackages) {

extensions/notebook/src/extension.ts

@@ -74,12 +74,8 @@ export async function activate(extensionContext: vscode.ExtensionContext): Promi
 		dialog.createDialog();
 	}));
 
-	extensionContext.subscriptions.push(vscode.commands.registerCommand('_notebook.command.new', async (context?: azdata.ConnectedContext) => {
+	extensionContext.subscriptions.push(vscode.commands.registerCommand('_notebook.command.new', async (options?: azdata.nb.NotebookShowOptions) => {
-		let connectionProfile: azdata.IConnectionProfile = undefined;
+		return appContext.notebookUtils.newNotebook(options);
-		if (context && context.connectionProfile) {
-			connectionProfile = context.connectionProfile;
-		}
-		return appContext.notebookUtils.newNotebook(connectionProfile);
 	}));
 	extensionContext.subscriptions.push(vscode.commands.registerCommand('notebook.command.open', async () => {
 		await appContext.notebookUtils.openNotebook();

extensions/notebook/src/integrationTest/notebookIntegration.test.ts

@@ -6,12 +6,10 @@
 import * as should from 'should';
 import * as vscode from 'vscode';
 import * as assert from 'assert';
-import * as path from 'path';
 import 'mocha';
 
 import { JupyterController } from '../jupyter/jupyterController';
 import { JupyterServerInstallation, PythonPkgDetails } from '../jupyter/jupyterServerInstallation';
-import { pythonBundleVersion } from '../common/constants';
 import { executeStreamedCommand, sortPackageVersions } from '../common/utils';
 
 describe('Notebook Extension Python Installation', function () {
@@ -59,16 +57,15 @@ describe('Notebook Extension Python Installation', function () {
 
 		console.log('Uninstalling existing pip dependencies');
 		let install = jupyterController.jupyterInstallation;
-		let pythonExe = JupyterServerInstallation.getPythonExePath(pythonInstallDir, false);
+		let pythonExe = JupyterServerInstallation.getPythonExePath(pythonInstallDir);
 		let command = `"${pythonExe}" -m pip uninstall -y jupyter pandas sparkmagic`;
 		await executeStreamedCommand(command, { env: install.execOptions.env }, install.outputChannel);
 		console.log('Uninstalling existing pip dependencies is done');
 
 		console.log('Start Existing Python Installation');
-		let existingPythonPath = path.join(pythonInstallDir, pythonBundleVersion);
+		await install.startInstallProcess(false, { installPath: pythonInstallDir, existingPython: true, packages: [] });
-		await install.startInstallProcess(false, { installPath: existingPythonPath, existingPython: true, packages: [] });
 		should(JupyterServerInstallation.isPythonInstalled()).be.true();
-		should(JupyterServerInstallation.getPythonInstallPath()).be.equal(existingPythonPath);
+		should(JupyterServerInstallation.getPythonInstallPath()).be.equal(pythonInstallDir);
 		should(JupyterServerInstallation.getExistingPythonSetting()).be.true();
 
 		// Redo "new" install to restore original settings.

extensions/notebook/src/jupyter/jupyterServerInstallation.ts

@@ -32,11 +32,18 @@ const msgInstallPkgStart = localize('msgInstallPkgStart', "Installing Notebook d
 const msgInstallPkgFinish = localize('msgInstallPkgFinish', "Notebook dependencies installation is complete");
 const msgPythonRunningError = localize('msgPythonRunningError', "Cannot overwrite an existing Python installation while python is running. Please close any active notebooks before proceeding.");
 const msgWaitingForInstall = localize('msgWaitingForInstall', "Another Python installation is currently in progress. Waiting for it to complete.");
+const msgShutdownJupyterNotebookSessions = localize('msgShutdownNotebookSessions', "Active Python notebook sessions will be shutdown in order to update. Would you like to proceed now?");
+function msgPythonVersionUpdatePrompt(pythonVersion: string): string { return localize('msgPythonVersionUpdatePrompt', "Python {0} is now available in Azure Data Studio. The current Python version (3.6.6) will be out of support in December 2021. Would you like to update to Python {0} now?", pythonVersion); }
+function msgPythonVersionUpdateWarning(pythonVersion: string): string { return localize('msgPythonVersionUpdateWarning', "Python {0} will be installed and will replace Python 3.6.6. Some packages may no longer be compatible with the new version or may need to be reinstalled. A notebook will be created to help you reinstall all pip packages. Would you like to continue with the update now?", pythonVersion); }
 function msgDependenciesInstallationFailed(errorMessage: string): string { return localize('msgDependenciesInstallationFailed', "Installing Notebook dependencies failed with error: {0}", errorMessage); }
 function msgDownloadPython(platform: string, pythonDownloadUrl: string): string { return localize('msgDownloadPython', "Downloading local python for platform: {0} to {1}", platform, pythonDownloadUrl); }
 function msgPackageRetrievalFailed(errorMessage: string): string { return localize('msgPackageRetrievalFailed', "Encountered an error when trying to retrieve list of installed packages: {0}", errorMessage); }
 function msgGetPythonUserDirFailed(errorMessage: string): string { return localize('msgGetPythonUserDirFailed', "Encountered an error when getting Python user path: {0}", errorMessage); }
 
+const yes = localize('yes', "Yes");
+const no = localize('no', "No");
+const dontAskAgain = localize('dontAskAgain', "Don't Ask Again");
+
 export interface PythonInstallSettings {
 	installPath: string;
 	existingPython: boolean;
@@ -110,6 +117,11 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 	private _usingConda: boolean;
 	private _installedPythonVersion: string;
 
+	private _upgradeInProcess: boolean = false;
+	private _oldPythonExecutable: string | undefined;
+	private _oldPythonInstallationPath: string | undefined;
+	private _oldUserInstalledPipPackages: PythonPkgDetails[] = [];
+
 	private _installInProgress: boolean;
 	private _installCompletion: Deferred<void>;
 
@@ -165,10 +177,41 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 
 		try {
 			let pythonExists = await utils.exists(this._pythonExecutable);
-			if (!pythonExists || forceInstall) {
+			let upgradePython = false;
+			// Warn users that some packages may need to be reinstalled after updating Python versions
+			if (!this._usingExistingPython && this._oldPythonExecutable && utils.compareVersions(await this.getInstalledPythonVersion(this._oldPythonExecutable), constants.pythonVersion) < 0) {
+				upgradePython = await vscode.window.showInformationMessage(msgPythonVersionUpdateWarning(constants.pythonVersion), yes, no) === yes;
+				if (upgradePython) {
+					this._upgradeInProcess = true;
+					if (await this.isPythonRunning(this._oldPythonExecutable)) {
+						let proceed = await vscode.window.showInformationMessage(msgShutdownJupyterNotebookSessions, yes, no) === yes;
+						if (!proceed) {
+							throw Error('Python update failed due to active Python notebook sessions.');
+						}
+						// Temporarily change the pythonExecutable to the old Python path so that the
+						// correct path is used to shutdown the old Python server.
+						let newPythonExecutable = this._pythonExecutable;
+						this._pythonExecutable = this._oldPythonExecutable;
+						await vscode.commands.executeCommand('notebook.action.stopJupyterNotebookSessions');
+						this._pythonExecutable = newPythonExecutable;
+					}
+
+					this._oldUserInstalledPipPackages = await this.getInstalledPipPackages(this._oldPythonExecutable, true);
+
+					if (await this.getInstalledPythonVersion(this._oldPythonExecutable) === '3.6.6') {
+						// Remove '0.0.1' from python executable path since the bundle version is removed from the path for ADS-Python 3.8.10+.
+						this._pythonExecutable = path.join(this._pythonInstallationPath, process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
+					}
+					await fs.remove(this._oldPythonInstallationPath).catch(err => {
+						throw (err);
+					});
+				}
+			}
+
+			if (!pythonExists || forceInstall || upgradePython) {
 				await this.installPythonPackage(backgroundOperation, this._usingExistingPython, this._pythonInstallationPath, this.outputChannel);
-				// reinstall pip to make sure !pip command works
+				// reinstall pip to make sure !pip command works on Windows
-				if (!this._usingExistingPython) {
+				if (!this._usingExistingPython && process.platform === constants.winPlatform) {
 					let packages: PythonPkgDetails[] = await this.getInstalledPipPackages(this._pythonExecutable);
 					let pip: PythonPkgDetails = packages.find(x => x.name === 'pip');
 					let cmd = `"${this._pythonExecutable}" -m pip install --force-reinstall pip=="${pip.version}"`;
@@ -191,14 +234,13 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 			return Promise.resolve();
 		}
 
-		let bundleVersion = constants.pythonBundleVersion;
 		let pythonVersion = constants.pythonVersion;
 		let platformId = utils.getOSPlatformId();
 		let packageName: string;
 		let pythonDownloadUrl: string;
 
 		let extension = process.platform === constants.winPlatform ? 'zip' : 'tar.gz';
-		packageName = `python-${pythonVersion}-${platformId}-${bundleVersion}.${extension}`;
+		packageName = `python-${pythonVersion}-${platformId}.${extension}`;
 
 		switch (process.platform) {
 			case constants.winPlatform:
@@ -257,16 +299,6 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 					.on('close', async () => {
 						//unpack python zip/tar file
 						outputChannel.appendLine(msgPythonUnpackPending);
-						let pythonSourcePath = path.join(installPath, constants.pythonBundleVersion);
-						if (await utils.exists(pythonSourcePath)) {
-							try {
-								// eslint-disable-next-line no-sync
-								fs.removeSync(pythonSourcePath);
-							} catch (err) {
-								backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonUnpackError);
-								return reject(err);
-							}
-						}
 						if (process.platform === constants.winPlatform) {
 							try {
 								let zippedFile = new zip(pythonPackagePathLocal);
@@ -319,16 +351,11 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		delete process.env['PYTHONSTARTUP'];
 		delete process.env['PYTHONHOME'];
 
-		//Python source path up to bundle version
-		let pythonSourcePath = this._usingExistingPython
-			? this._pythonInstallationPath
-			: path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
-
 		// Update python paths and properties to reference user's local python.
 		let pythonBinPathSuffix = process.platform === constants.winPlatform ? '' : 'bin';
 
-		this._pythonExecutable = JupyterServerInstallation.getPythonExePath(this._pythonInstallationPath, this._usingExistingPython);
+		this._pythonExecutable = JupyterServerInstallation.getPythonExePath(this._pythonInstallationPath);
-		this.pythonBinPath = path.join(pythonSourcePath, pythonBinPathSuffix);
+		this.pythonBinPath = path.join(this._pythonInstallationPath, pythonBinPathSuffix);
 
 		this._usingConda = this.isCondaInstalled();
 
@@ -338,15 +365,15 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		let delimiter = path.delimiter;
 		this.pythonEnvVarPath = this.pythonBinPath + delimiter + this.pythonEnvVarPath;
 		if (process.platform === constants.winPlatform) {
-			let pythonScriptsPath = path.join(pythonSourcePath, 'Scripts');
+			let pythonScriptsPath = path.join(this._pythonInstallationPath, 'Scripts');
 			this.pythonEnvVarPath = pythonScriptsPath + delimiter + this.pythonEnvVarPath;
 
 			if (this._usingConda) {
 				this.pythonEnvVarPath = [
-					path.join(pythonSourcePath, 'Library', 'mingw-w64', 'bin'),
+					path.join(this._pythonInstallationPath, 'Library', 'mingw-w64', 'bin'),
-					path.join(pythonSourcePath, 'Library', 'usr', 'bin'),
+					path.join(this._pythonInstallationPath, 'Library', 'usr', 'bin'),
-					path.join(pythonSourcePath, 'Library', 'bin'),
+					path.join(this._pythonInstallationPath, 'Library', 'bin'),
-					path.join(pythonSourcePath, 'condabin'),
+					path.join(this._pythonInstallationPath, 'condabin'),
 					this.pythonEnvVarPath
 				].join(delimiter);
 			}
@@ -405,7 +432,7 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		// This step is skipped when using an existing installation or when upgrading
 		// packages, since those cases wouldn't overwrite the installation.
 		if (!installSettings.existingPython && !installSettings.packageUpgradeOnly) {
-			let pythonExePath = JupyterServerInstallation.getPythonExePath(installSettings.installPath, false);
+			let pythonExePath = JupyterServerInstallation.getPythonExePath(installSettings.installPath);
 			let isPythonRunning = await this.isPythonRunning(pythonExePath);
 			if (isPythonRunning) {
 				return Promise.reject(msgPythonRunningError);
@@ -438,7 +465,21 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 
 						this._installCompletion.resolve();
 						this._installInProgress = false;
-						await vscode.commands.executeCommand('notebook.action.restartJupyterNotebookSessions');
+						if (this._upgradeInProcess) {
+							// Pass in false for restartJupyterServer parameter since the jupyter server has already been shutdown
+							// when removing the old Python version on Windows.
+							if (process.platform === constants.winPlatform) {
+								await vscode.commands.executeCommand('notebook.action.restartJupyterNotebookSessions', false);
+							} else {
+								await vscode.commands.executeCommand('notebook.action.restartJupyterNotebookSessions');
+							}
+							if (this._oldUserInstalledPipPackages.length !== 0) {
+								await this.createInstallPipPackagesHelpNotebook(this._oldUserInstalledPipPackages);
+							}
+							this._upgradeInProcess = false;
+						} else {
+							await vscode.commands.executeCommand('notebook.action.restartJupyterNotebookSessions');
+						}
 					})
 					.catch(err => {
 						let errorMsg = msgDependenciesInstallationFailed(utils.getErrorMessage(err));
@@ -464,6 +505,12 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		}
 
 		let isPythonInstalled = JupyterServerInstallation.isPythonInstalled();
+
+		// If the latest version of ADS-Python is not installed, then prompt the user to upgrade
+		if (isPythonInstalled && !this._usingExistingPython && utils.compareVersions(await this.getInstalledPythonVersion(this._pythonExecutable), constants.pythonVersion) < 0) {
+			this.promptUserForPythonUpgrade();
+		}
+
 		let areRequiredPackagesInstalled = await this.areRequiredPackagesInstalled(kernelDisplayName);
 		if (!isPythonInstalled || !areRequiredPackagesInstalled) {
 			let pythonWizard = new ConfigurePythonWizard(this);
@@ -474,6 +521,22 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		}
 	}
 
+	private async promptUserForPythonUpgrade(): Promise<void> {
+		let notebookConfig: vscode.WorkspaceConfiguration = vscode.workspace.getConfiguration(constants.notebookConfigKey);
+		if (notebookConfig && notebookConfig[constants.dontPromptPythonUpdate]) {
+			return;
+		}
+
+		let response = await vscode.window.showInformationMessage(msgPythonVersionUpdatePrompt(constants.pythonVersion), yes, no, dontAskAgain);
+		if (response === yes) {
+			this._oldPythonInstallationPath = path.join(this._pythonInstallationPath);
+			this._oldPythonExecutable = this._pythonExecutable;
+			vscode.commands.executeCommand(constants.jupyterConfigurePython);
+		} else if (response === dontAskAgain) {
+			await notebookConfig.update(constants.dontPromptPythonUpdate, true, vscode.ConfigurationTarget.Global);
+		}
+	}
+
 	private async areRequiredPackagesInstalled(kernelDisplayName: string): Promise<boolean> {
 		if (this._kernelSetupCache.get(kernelDisplayName)) {
 			return true;
@@ -487,7 +550,7 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		let requiredPackages = this.getRequiredPackagesForKernel(kernelDisplayName);
 		for (let pkg of requiredPackages) {
 			let installedVersion = installedPackageMap.get(pkg.name);
-			if (!installedVersion || utils.comparePackageVersions(installedVersion, pkg.version) < 0) {
+			if (!installedVersion || utils.compareVersions(installedVersion, pkg.version) < 0) {
 				return false;
 			}
 		}
@@ -508,7 +571,7 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 
 			packages.forEach(pkg => {
 				let installedPkgVersion = pipVersionMap.get(pkg.name);
-				if (!installedPkgVersion || utils.comparePackageVersions(installedPkgVersion, pkg.version) < 0) {
+				if (!installedPkgVersion || utils.compareVersions(installedPkgVersion, pkg.version) < 0) {
 					packagesToInstall.push(pkg);
 				}
 			});
@@ -520,7 +583,7 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		}
 	}
 
-	public async getInstalledPipPackages(pythonExePath?: string): Promise<PythonPkgDetails[]> {
+	public async getInstalledPipPackages(pythonExePath?: string, checkUserPackages: boolean = false): Promise<PythonPkgDetails[]> {
 		try {
 			if (pythonExePath) {
 				if (!fs.existsSync(pythonExePath)) {
@@ -531,6 +594,9 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 			}
 
 			let cmd = `"${pythonExePath ?? this.pythonExecutable}" -m pip list --format=json`;
+			if (checkUserPackages) {
+				cmd = cmd.concat(' --user');
+			}
 			let packagesInfo = await this.executeBufferedCommand(cmd);
 			let packages: PythonPkgDetails[] = [];
 			if (packagesInfo) {
@@ -676,8 +742,7 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 			return false;
 		}
 
-		let useExistingInstall = JupyterServerInstallation.getExistingPythonSetting();
+		let pythonExe = JupyterServerInstallation.getPythonExePath(pathSetting);
-		let pythonExe = JupyterServerInstallation.getPythonExePath(pathSetting, useExistingInstall);
 		// eslint-disable-next-line no-sync
 		return fs.existsSync(pythonExe);
 	}
@@ -713,11 +778,25 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		return path;
 	}
 
-	public static getPythonExePath(pythonInstallPath: string, useExistingInstall: boolean): string {
+	public static getPythonExePath(pythonInstallPath: string): string {
-		return path.join(
+		// The bundle version (0.0.1) is removed from the path for ADS-Python 3.8.10+.
+		// Only ADS-Python 3.6.6 contains the bundle version in the path.
+		let oldPythonPath = path.join(
 			pythonInstallPath,
-			useExistingInstall ? '' : constants.pythonBundleVersion,
+			'0.0.1',
 			process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
+		let newPythonPath = path.join(
+			pythonInstallPath,
+			process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
+
+		// Note: If Python exists in both paths (which can happen if the user chose not to remove Python 3.6 when upgrading),
+		// then we want to default to using the newer Python version.
+		if (!fs.existsSync(newPythonPath) && !fs.existsSync(oldPythonPath) || fs.existsSync(newPythonPath)) {
+			return newPythonPath;
+		}
+		// If Python only exists in the old path then return the old path.
+		// This is for users who are still using Python 3.6
+		return oldPythonPath;
 	}
 
 	private async getPythonUserDir(pythonExecutable: string): Promise<string> {
@@ -780,6 +859,37 @@ export class JupyterServerInstallation implements IJupyterServerInstallation {
 		kernelSpec.argv = kernelSpec.argv?.map(arg => arg.replace('{ADS_PYTHONDIR}', this._pythonInstallationPath));
 		await fs.writeFile(kernelPath, JSON.stringify(kernelSpec, undefined, '\t'));
 	}
+
+	private async createInstallPipPackagesHelpNotebook(userInstalledPipPackages: PythonPkgDetails[]): Promise<void> {
+		let packagesList: string[] = userInstalledPipPackages.map(pkg => { return pkg.name; });
+		let installPackagesCode = `import sys\n!{sys.executable} -m pip install --user ${packagesList.join(' ')}`;
+		let initialContent: azdata.nb.INotebookContents = {
+			cells: [{
+				cell_type: 'markdown',
+				source: ['# Install Pip Packages\n\nThis notebook will help you reinstall the pip packages you were previously using so that they can be used with Python 3.8.\n\n**Note:** Some packages may have a dependency on Python 3.6 and will not work with Python 3.8.\n\nRun the following code cell after Python 3.8 installation is complete.'],
+			}, {
+				cell_type: 'code',
+				source: [installPackagesCode],
+			}],
+			metadata: {
+				kernelspec: {
+					name: 'python3',
+					language: 'python3',
+					display_name: 'Python 3'
+				},
+				language_info: {
+					name: 'python3'
+				}
+			},
+			nbformat: 4,
+			nbformat_minor: 5
+		};
+
+		await vscode.commands.executeCommand('_notebook.command.new', {
+			initialContent: JSON.stringify(initialContent),
+			defaultKernel: 'Python 3'
+		});
+	}
 }
 
 export interface PythonPkgDetails {

extensions/notebook/src/test/common/utils.test.ts

@@ -49,50 +49,50 @@ describe('Utils Tests', function () {
 		should(utils.getOSPlatformId()).not.throw();
 	});
 
-	describe('comparePackageVersions', () => {
+	describe('compareVersions', () => {
 		const version1 = '1.0.0.0';
 		const version1Revision = '1.0.0.1';
 		const version2 = '2.0.0.0';
 		const shortVersion1 = '1';
 
 		it('same id', () => {
-			should(utils.comparePackageVersions(version1, version1)).equal(0);
+			should(utils.compareVersions(version1, version1)).equal(0);
 		});
 
 		it('first version lower', () => {
-			should(utils.comparePackageVersions(version1, version2)).equal(-1);
+			should(utils.compareVersions(version1, version2)).equal(-1);
 		});
 
 		it('second version lower', () => {
-			should(utils.comparePackageVersions(version2, version1)).equal(1);
+			should(utils.compareVersions(version2, version1)).equal(1);
 		});
 
 		it('short first version is padded correctly', () => {
-			should(utils.comparePackageVersions(shortVersion1, version1)).equal(0);
+			should(utils.compareVersions(shortVersion1, version1)).equal(0);
 		});
 
 		it('short second version is padded correctly when', () => {
-			should(utils.comparePackageVersions(version1, shortVersion1)).equal(0);
+			should(utils.compareVersions(version1, shortVersion1)).equal(0);
 		});
 
 		it('correctly compares version with only minor version difference', () => {
-			should(utils.comparePackageVersions(version1Revision, version1)).equal(1);
+			should(utils.compareVersions(version1Revision, version1)).equal(1);
 		});
 
 		it('equivalent versions with wildcard characters', () => {
-			should(utils.comparePackageVersions('1.*.3', '1.5.3')).equal(0);
+			should(utils.compareVersions('1.*.3', '1.5.3')).equal(0);
 		});
 
 		it('lower version with wildcard characters', () => {
-			should(utils.comparePackageVersions('1.4.*', '1.5.3')).equal(-1);
+			should(utils.compareVersions('1.4.*', '1.5.3')).equal(-1);
 		});
 
 		it('higher version with wildcard characters', () => {
-			should(utils.comparePackageVersions('4.5.6', '3.*')).equal(1);
+			should(utils.compareVersions('4.5.6', '3.*')).equal(1);
 		});
 
 		it('all wildcard strings should be equal', () => {
-			should(utils.comparePackageVersions('*.*', '*.*.*')).equal(0);
+			should(utils.compareVersions('*.*', '*.*.*')).equal(0);
 		});
 	});
 

src/sql/workbench/contrib/notebook/browser/notebook.contribution.ts

@@ -174,7 +174,7 @@ const RESTART_JUPYTER_NOTEBOOK_SESSIONS = 'notebook.action.restartJupyterNoteboo
 
 CommandsRegistry.registerCommand({
 	id: RESTART_JUPYTER_NOTEBOOK_SESSIONS,
-	handler: async (accessor: ServicesAccessor) => {
+	handler: async (accessor: ServicesAccessor, restartJupyterServer: boolean = true) => {
 		const editorService: IEditorService = accessor.get(IEditorService);
 		const editors: readonly IEditorInput[] = editorService.editors;
 		let jupyterServerRestarted: boolean = false;
@@ -184,7 +184,7 @@ CommandsRegistry.registerCommand({
 				let model: INotebookModel = editor.notebookModel;
 				if (model.providerId === 'jupyter' && model.clientSession.isReady) {
 					// Jupyter server needs to be restarted so that the correct Python installation is used
-					if (!jupyterServerRestarted) {
+					if (!jupyterServerRestarted && restartJupyterServer) {
 						let jupyterNotebookManager: INotebookManager = model.notebookManagers.find(x => x.providerId === 'jupyter');
 						// Shutdown all current Jupyter sessions before stopping the server
 						await jupyterNotebookManager.sessionManager.shutdownAll();
@@ -204,6 +204,29 @@ CommandsRegistry.registerCommand({
 	}
 });
 
+const STOP_JUPYTER_NOTEBOOK_SESSIONS = 'notebook.action.stopJupyterNotebookSessions';
+
+CommandsRegistry.registerCommand({
+	id: STOP_JUPYTER_NOTEBOOK_SESSIONS,
+	handler: async (accessor: ServicesAccessor) => {
+		const editorService: IEditorService = accessor.get(IEditorService);
+		const editors: readonly IEditorInput[] = editorService.editors;
+
+		for (let editor of editors) {
+			if (editor instanceof NotebookInput) {
+				let model: INotebookModel = editor.notebookModel;
+				if (model?.providerId === 'jupyter') {
+					let jupyterNotebookManager: INotebookManager = model.notebookManagers.find(x => x.providerId === 'jupyter');
+					await jupyterNotebookManager.sessionManager.shutdownAll();
+					jupyterNotebookManager.sessionManager.dispose();
+					await jupyterNotebookManager.serverManager.stopServer();
+					return;
+				}
+			}
+		}
+	}
+});
+
 MenuRegistry.appendMenuItem(MenuId.CommandPalette, {
 	command: {
 		id: TOGGLE_TAB_FOCUS_COMMAND_ID,