diff --git a/extensions/integration-tests/src/notebook.test.ts b/extensions/integration-tests/src/notebook.test.ts index bc7c1982f9..6c502b93b3 100644 --- a/extensions/integration-tests/src/notebook.test.ts +++ b/extensions/integration-tests/src/notebook.test.ts @@ -10,7 +10,7 @@ import * as assert from 'assert'; import * as azdata from 'azdata'; import * as vscode from 'vscode'; import { context } from './testContext'; -import { sqlNotebookContent, writeNotebookToFile, sqlKernelMetadata, getFileName, pySparkNotebookContent, pySpark3KernelMetadata, pythonKernelMetadata, sqlNotebookMultipleCellsContent, notebookContentForCellLanguageTest, sqlKernelSpec, pythonKernelSpec, pySpark3KernelSpec, CellTypes } from './notebook.util'; +import { sqlNotebookContent, writeNotebookToFile, sqlKernelMetadata, getFileName, pySparkNotebookContent, pySparkKernelMetadata, pythonKernelMetadata, sqlNotebookMultipleCellsContent, notebookContentForCellLanguageTest, sqlKernelSpec, pythonKernelSpec, pySparkKernelSpec, CellTypes } from './notebook.util'; import { getBdcServer, getConfigValue, EnvironmentVariable_PYTHON_PATH } from './testConfig'; import { connectToServer, sleep } from './utils'; import * as fs from 'fs'; @@ -79,14 +79,14 @@ if (context.RunTest) { await (new NotebookTester()).pythonChangeKernelDifferentProviderTest(this.test.title); }); - test('Change kernel same provider Python to PySpark3 to Python', async function () { + test('Change kernel same provider Python to PySpark to Python', async function () { await (new NotebookTester()).pythonChangeKernelSameProviderTest(this.test.title); }); } if (process.env['RUN_PYSPARK_TEST'] === '1') { - test('PySpark3 notebook test', async function () { - await (new NotebookTester()).pySpark3NbTest(this.test.title); + test('PySpark notebook test', async function () { + await (new NotebookTester()).pySparkNbTest(this.test.title); }); } @@ -112,8 +112,8 @@ class NotebookTester { invocationCount: number = 0; @stressify({ dop: NotebookTester.ParallelCount }) - async pySpark3NbTest(title: string): Promise { - let notebook = await this.openNotebook(pySparkNotebookContent, pySpark3KernelMetadata, title + this.invocationCount++); + async pySparkNbTest(title: string): Promise { + let notebook = await this.openNotebook(pySparkNotebookContent, pySparkKernelMetadata, title + this.invocationCount++); await this.runCell(notebook); let cellOutputs = notebook.document.cells[0].contents.outputs; let sparkResult = (cellOutputs[3]).text; @@ -281,9 +281,9 @@ class NotebookTester { assert(notebook.document.providerId === 'jupyter', `Expected providerId to be jupyter, Actual: ${notebook.document.providerId}`); assert(notebook.document.kernelSpec.name === 'python3', `Expected first kernel name: python3, Actual: ${notebook.document.kernelSpec.name}`); - let kernelChanged = await notebook.changeKernel(pySpark3KernelSpec); + let kernelChanged = await notebook.changeKernel(pySparkKernelSpec); assert(notebook.document.providerId === 'jupyter', `Expected providerId to be jupyter, Actual: ${notebook.document.providerId}`); - assert(kernelChanged && notebook.document.kernelSpec.name === 'pyspark3kernel', `Expected second kernel name: pyspark3kernel, Actual: ${notebook.document.kernelSpec.name}`); + assert(kernelChanged && notebook.document.kernelSpec.name === 'pysparkkernel', `Expected second kernel name: pysparkkernel, Actual: ${notebook.document.kernelSpec.name}`); kernelChanged = await notebook.changeKernel(pythonKernelSpec); assert(notebook.document.providerId === 'jupyter', `Expected providerId to be jupyter, Actual: ${notebook.document.providerId}`); diff --git a/extensions/integration-tests/src/notebook.util.ts b/extensions/integration-tests/src/notebook.util.ts index 976d192439..64393781e8 100644 --- a/extensions/integration-tests/src/notebook.util.ts +++ b/extensions/integration-tests/src/notebook.util.ts @@ -27,8 +27,8 @@ export const pySparkNotebookContent: azdata.nb.INotebookContents = { }], metadata: { 'kernelspec': { - 'name': 'pyspark3kernel', - 'display_name': 'PySpark3' + 'name': 'pysparkkernel', + 'display_name': 'PySpark' } }, nbformat: 4, @@ -127,16 +127,16 @@ export const sqlNotebookMultipleCellsContent: azdata.nb.INotebookContents = { nbformat_minor: 2 }; -export const pySpark3KernelMetadata = { +export const pySparkKernelMetadata = { 'kernelspec': { - 'name': 'pyspark3kernel', - 'display_name': 'PySpark3' + 'name': 'pysparkkernel', + 'display_name': 'PySpark' } }; -export const pySpark3KernelSpec = { - name: 'pyspark3', - display_name: 'PySpark3' +export const pySparkKernelSpec = { + name: 'pyspark', + display_name: 'PySpark' }; export const sqlKernelMetadata = { diff --git a/extensions/notebook/kernels/pyspark3kernel/kernel.json b/extensions/notebook/kernels/pyspark3kernel/kernel.json deleted file mode 100644 index 5e77ed1448..0000000000 --- a/extensions/notebook/kernels/pyspark3kernel/kernel.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "argv": [ - "python", - "-m", - "sparkmagic.kernels.pyspark3kernel.pyspark3kernel", - "-f", - "{connection_file}" - ], - "display_name": "PySpark3" -} \ No newline at end of file diff --git a/extensions/notebook/package.json b/extensions/notebook/package.json index b1d7d51db9..1282b788d1 100644 --- a/extensions/notebook/package.json +++ b/extensions/notebook/package.json @@ -310,14 +310,6 @@ "IPYNB" ], "standardKernels": [ - { - "name": "pyspark3kernel", - "displayName": "PySpark3", - "connectionProviderIds": [ - "HADOOP_KNOX", - "MSSQL" - ] - }, { "name": "pysparkkernel", "displayName": "PySpark", diff --git a/extensions/notebook/src/common/constants.ts b/extensions/notebook/src/common/constants.ts index 99685d824a..a64b8c634f 100644 --- a/extensions/notebook/src/common/constants.ts +++ b/extensions/notebook/src/common/constants.ts @@ -11,7 +11,6 @@ export const extensionOutputChannel = 'Notebooks'; // JUPYTER CONFIG ////////////////////////////////////////////////////////// export const pythonBundleVersion = '0.0.1'; export const pythonVersion = '3.6.6'; -export const sparkMagicVersion = '0.12.6.1'; export const pythonPathConfigKey = 'pythonPath'; export const existingPythonConfigKey = 'useExistingPython'; export const notebookConfigKey = 'notebook'; @@ -42,7 +41,6 @@ export enum PythonPkgType { Anaconda = 'Anaconda' } -export const pythonOfflinePipPackagesUrl = 'https://go.microsoft.com/fwlink/?linkid=2092867'; -export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092866'; -export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092865'; -export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092864'; +export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103837'; +export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103836'; +export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103835'; diff --git a/extensions/notebook/src/extension.ts b/extensions/notebook/src/extension.ts index 9b8fafd5a6..15b0a40ef9 100644 --- a/extensions/notebook/src/extension.ts +++ b/extensions/notebook/src/extension.ts @@ -221,8 +221,8 @@ async function analyzeNotebook(oeContext?: azdata.ObjectExplorerContext): Promis providerId: JUPYTER_NOTEBOOK_PROVIDER, preview: false, defaultKernel: { - name: 'pyspark3kernel', - display_name: 'PySpark3', + name: 'pysparkkernel', + display_name: 'PySpark', language: 'python' } }); diff --git a/extensions/notebook/src/jupyter/jupyterController.ts b/extensions/notebook/src/jupyter/jupyterController.ts index 87809467b3..d68767d5c2 100644 --- a/extensions/notebook/src/jupyter/jupyterController.ts +++ b/extensions/notebook/src/jupyter/jupyterController.ts @@ -150,8 +150,8 @@ export class JupyterController implements vscode.Disposable { providerId: constants.jupyterNotebookProviderId, preview: false, defaultKernel: { - name: 'pyspark3kernel', - display_name: 'PySpark3', + name: 'pysparkkernel', + display_name: 'PySpark', language: 'python' } }); diff --git a/extensions/notebook/src/jupyter/jupyterServerInstallation.ts b/extensions/notebook/src/jupyter/jupyterServerInstallation.ts index 97dd6e1a4a..f77ce88121 100644 --- a/extensions/notebook/src/jupyter/jupyterServerInstallation.ts +++ b/extensions/notebook/src/jupyter/jupyterServerInstallation.ts @@ -77,9 +77,6 @@ export class JupyterServerInstallation { try { await this.installPythonPackage(backgroundOperation); - this.outputChannel.appendLine(msgPythonDownloadComplete); - backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonDownloadComplete); - if (this._usingConda) { await this.installCondaDependencies(); } else if (this._usingExistingPython) { @@ -87,19 +84,11 @@ export class JupyterServerInstallation { } else { await this.installOfflinePipDependencies(); } - let doOnlineInstall = this._usingExistingPython; - await this.installSparkMagic(doOnlineInstall); } catch (err) { this.outputChannel.appendLine(msgDependenciesInstallationFailed(utils.getErrorMessage(err))); throw err; } - fs.remove(this._pythonPackageDir, (err: Error) => { - if (err) { - this.outputChannel.appendLine(err.message); - } - }); - this.outputChannel.appendLine(msgInstallPkgFinish); backgroundOperation.updateStatus(azdata.TaskStatus.Succeeded, msgInstallPkgFinish); window.showInformationMessage(msgInstallPkgFinish); @@ -107,41 +96,34 @@ export class JupyterServerInstallation { } private installPythonPackage(backgroundOperation: azdata.BackgroundOperation): Promise { + if (this._usingExistingPython) { + return Promise.resolve(); + } + let bundleVersion = constants.pythonBundleVersion; let pythonVersion = constants.pythonVersion; let platformId = utils.getOSPlatformId(); let packageName: string; let pythonDownloadUrl: string; - if (this._usingExistingPython) { - packageName = `python-${pythonVersion}-${bundleVersion}-offlinePackages.zip`; - pythonDownloadUrl = constants.pythonOfflinePipPackagesUrl; - } else { - let extension = process.platform === constants.winPlatform ? 'zip' : 'tar.gz'; - packageName = `python-${pythonVersion}-${platformId}-${bundleVersion}.${extension}`; - switch (utils.getOSPlatform()) { - case utils.Platform.Windows: - pythonDownloadUrl = constants.pythonWindowsInstallUrl; - break; - case utils.Platform.Mac: - pythonDownloadUrl = constants.pythonMacInstallUrl; - break; - default: - // Default to linux - pythonDownloadUrl = constants.pythonLinuxInstallUrl; - break; - } + let extension = process.platform === constants.winPlatform ? 'zip' : 'tar.gz'; + packageName = `python-${pythonVersion}-${platformId}-${bundleVersion}.${extension}`; + + switch (utils.getOSPlatform()) { + case utils.Platform.Windows: + pythonDownloadUrl = constants.pythonWindowsInstallUrl; + break; + case utils.Platform.Mac: + pythonDownloadUrl = constants.pythonMacInstallUrl; + break; + default: + // Default to linux + pythonDownloadUrl = constants.pythonLinuxInstallUrl; + break; } - let installPath: string; - if (this._usingExistingPython) { - installPath = utils.getUserHome(); - } else { - installPath = this._pythonInstallationPath; - } - - let pythonPackagePathLocal = path.join(installPath, packageName); return new Promise((resolve, reject) => { + let installPath = this._pythonInstallationPath; backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgDownloadPython(platformId, pythonDownloadUrl)); fs.mkdirs(installPath, (err) => { if (err) { @@ -179,12 +161,13 @@ export class JupyterServerInstallation { } }); + let pythonPackagePathLocal = path.join(installPath, packageName); downloadRequest.pipe(fs.createWriteStream(pythonPackagePathLocal)) .on('close', async () => { //unpack python zip/tar file this.outputChannel.appendLine(msgPythonUnpackPending); let pythonSourcePath = path.join(installPath, constants.pythonBundleVersion); - if (!this._usingExistingPython && await utils.exists(pythonSourcePath)) { + if (await utils.exists(pythonSourcePath)) { try { fs.removeSync(pythonSourcePath); } catch (err) { @@ -201,6 +184,8 @@ export class JupyterServerInstallation { } }); + this.outputChannel.appendLine(msgPythonDownloadComplete); + backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonDownloadComplete); resolve(); }).catch(err => { backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonUnpackError); @@ -223,7 +208,7 @@ export class JupyterServerInstallation { : path.join(this._pythonInstallationPath, constants.pythonBundleVersion); if (this._usingExistingPython) { - this._pythonPackageDir = path.join(utils.getUserHome(), 'offlinePackages'); + this._pythonPackageDir = undefined; } else { this._pythonPackageDir = path.join(pythonSourcePath, 'offlinePackages'); } @@ -438,41 +423,27 @@ export class JupyterServerInstallation { installJupyterCommand = `"${this._pythonExecutable}" -m pip install --force-reinstall --no-index pip --find-links "${this._pythonPackageDir}" --no-warn-script-location`; await this.executeStreamedCommand(installJupyterCommand); + fs.remove(this._pythonPackageDir, (err: Error) => { + if (err) { + this.outputChannel.appendLine(err.message); + } + }); + this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete.")); } else { return Promise.resolve(); } } - private async installSparkMagic(doOnlineInstall: boolean): Promise { - let installSparkMagic: string; - if (process.platform === constants.winPlatform || this._usingExistingPython) { - // Overwrite existing install of sparkmagic, since we use a custom version - let cmdOptions = this._usingExistingPython ? '--user --force-reinstall' : '--force-reinstall'; - let sparkWheel = path.join(this._pythonPackageDir, `sparkmagic-${constants.sparkMagicVersion}-py3-none-any.whl`); - if (doOnlineInstall) { - installSparkMagic = `"${this._pythonExecutable}" -m pip install ${cmdOptions} "${sparkWheel}" --no-warn-script-location`; - } else { - installSparkMagic = `"${this._pythonExecutable}" -m pip install ${cmdOptions} --no-index "${sparkWheel}" --find-links "${this._pythonPackageDir}" --no-warn-script-location`; - } - } - - if (installSparkMagic) { - this.outputChannel.show(true); - this.outputChannel.appendLine(localize('msgInstallingSpark', "Installing SparkMagic...")); - await this.executeStreamedCommand(installSparkMagic); - } - } - private async installPipDependencies(): Promise { this.outputChannel.show(true); this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks...")); let cmdOptions = this._usingExistingPython ? '--user' : ''; - let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} jupyter==1.0.0 pandas==0.24.2`; + let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} jupyter>=1.0.0 pandas>=0.24.2 sparkmagic>=0.12.9`; await this.executeStreamedCommand(installCommand); - installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`; + installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator>=1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`; await this.executeStreamedCommand(installCommand); this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete.")); @@ -482,14 +453,14 @@ export class JupyterServerInstallation { this.outputChannel.show(true); this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks...")); - let installCommand = `"${this.getCondaExePath()}" install -y jupyter==1.0.0 pandas==0.24.2`; + let installCommand = `"${this.getCondaExePath()}" install -y jupyter>=1.0.0 pandas>=0.24.2`; if (process.platform !== constants.winPlatform) { - installCommand = `${installCommand} pykerberos==1.2.1`; + installCommand = `${installCommand} pykerberos>=1.2.1`; } await this.executeStreamedCommand(installCommand); let cmdOptions = this._usingExistingPython ? '--user' : ''; - installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`; + installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`; await this.executeStreamedCommand(installCommand); this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete.")); diff --git a/src/sql/workbench/services/notebook/common/notebookRegistry.ts b/src/sql/workbench/services/notebook/common/notebookRegistry.ts index 8d84f8c9d2..e14faf6ba7 100644 --- a/src/sql/workbench/services/notebook/common/notebookRegistry.ts +++ b/src/sql/workbench/services/notebook/common/notebookRegistry.ts @@ -113,7 +113,7 @@ let notebookLanguageMagicType: IJSONSchema = { type: 'string' }, kernels: { - description: localize('carbon.extension.contributes.notebook.kernels', "Optional set of kernels this is valid for, e.g. python3, pyspark3, sql"), + description: localize('carbon.extension.contributes.notebook.kernels', "Optional set of kernels this is valid for, e.g. python3, pyspark, sql"), oneOf: [ { type: 'string' }, {