ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 18:46:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Update python packages to use sparkmagic 0.12.9 (#7240)

Browse Source
This commit is contained in:
Cory Rivera
2019-09-17 15:05:59 -07:00
committed by GitHub
parent 290dd9531f
commit 094d6f2339
9 changed files with 59 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
extensions/notebook/kernels/pyspark3kernel/kernel.json
View File

@@ -1,10 +0,0 @@
{
"argv": [
"python",
"-m",
"sparkmagic.kernels.pyspark3kernel.pyspark3kernel",
"-f",
"{connection_file}"
],
"display_name": "PySpark3"
}

8
extensions/notebook/package.json
View File

@@ -310,14 +310,6 @@
"IPYNB"
],
"standardKernels": [
{
"name": "pyspark3kernel",
"displayName": "PySpark3",
"connectionProviderIds": [
"HADOOP_KNOX",
"MSSQL"
]
},
{
"name": "pysparkkernel",
"displayName": "PySpark",

8
extensions/notebook/src/common/constants.ts
View File

@@ -11,7 +11,6 @@ export const extensionOutputChannel = 'Notebooks';
// JUPYTER CONFIG //////////////////////////////////////////////////////////
export const pythonBundleVersion = '0.0.1';
export const pythonVersion = '3.6.6';
export const sparkMagicVersion = '0.12.6.1';
export const pythonPathConfigKey = 'pythonPath';
export const existingPythonConfigKey = 'useExistingPython';
export const notebookConfigKey = 'notebook';
@@ -42,7 +41,6 @@ export enum PythonPkgType {
Anaconda = 'Anaconda'
}
export const pythonOfflinePipPackagesUrl = 'https://go.microsoft.com/fwlink/?linkid=2092867';
export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092866';
export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092865';
export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2092864';
export const pythonWindowsInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103837';
export const pythonMacInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103836';
export const pythonLinuxInstallUrl = 'https://go.microsoft.com/fwlink/?linkid=2103835';

4
extensions/notebook/src/extension.ts
View File

@@ -221,8 +221,8 @@ async function analyzeNotebook(oeContext?: azdata.ObjectExplorerContext): Promis
providerId: JUPYTER_NOTEBOOK_PROVIDER,
preview: false,
defaultKernel: {
name: 'pyspark3kernel',
display_name: 'PySpark3',
name: 'pysparkkernel',
display_name: 'PySpark',
language: 'python'
}
});

4
extensions/notebook/src/jupyter/jupyterController.ts
View File

@@ -150,8 +150,8 @@ export class JupyterController implements vscode.Disposable {
providerId: constants.jupyterNotebookProviderId,
preview: false,
defaultKernel: {
name: 'pyspark3kernel',
display_name: 'PySpark3',
name: 'pysparkkernel',
display_name: 'PySpark',
language: 'python'
}
});

99
extensions/notebook/src/jupyter/jupyterServerInstallation.ts
View File

@@ -77,9 +77,6 @@ export class JupyterServerInstallation {
try {
await this.installPythonPackage(backgroundOperation);
this.outputChannel.appendLine(msgPythonDownloadComplete);
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonDownloadComplete);
if (this._usingConda) {
await this.installCondaDependencies();
} else if (this._usingExistingPython) {
@@ -87,19 +84,11 @@ export class JupyterServerInstallation {
} else {
await this.installOfflinePipDependencies();
}
let doOnlineInstall = this._usingExistingPython;
await this.installSparkMagic(doOnlineInstall);
} catch (err) {
this.outputChannel.appendLine(msgDependenciesInstallationFailed(utils.getErrorMessage(err)));
throw err;
}
fs.remove(this._pythonPackageDir, (err: Error) => {
if (err) {
this.outputChannel.appendLine(err.message);
}
});
this.outputChannel.appendLine(msgInstallPkgFinish);
backgroundOperation.updateStatus(azdata.TaskStatus.Succeeded, msgInstallPkgFinish);
window.showInformationMessage(msgInstallPkgFinish);
@@ -107,41 +96,34 @@ export class JupyterServerInstallation {
}
private installPythonPackage(backgroundOperation: azdata.BackgroundOperation): Promise<void> {
if (this._usingExistingPython) {
return Promise.resolve();
}
let bundleVersion = constants.pythonBundleVersion;
let pythonVersion = constants.pythonVersion;
let platformId = utils.getOSPlatformId();
let packageName: string;
let pythonDownloadUrl: string;
if (this._usingExistingPython) {
packageName = `python-${pythonVersion}-${bundleVersion}-offlinePackages.zip`;
pythonDownloadUrl = constants.pythonOfflinePipPackagesUrl;
} else {
let extension = process.platform === constants.winPlatform ? 'zip' : 'tar.gz';
packageName = `python-${pythonVersion}-${platformId}-${bundleVersion}.${extension}`;
switch (utils.getOSPlatform()) {
case utils.Platform.Windows:
pythonDownloadUrl = constants.pythonWindowsInstallUrl;
break;
case utils.Platform.Mac:
pythonDownloadUrl = constants.pythonMacInstallUrl;
break;
default:
// Default to linux
pythonDownloadUrl = constants.pythonLinuxInstallUrl;
break;
}
let extension = process.platform === constants.winPlatform ? 'zip' : 'tar.gz';
packageName = `python-${pythonVersion}-${platformId}-${bundleVersion}.${extension}`;
switch (utils.getOSPlatform()) {
case utils.Platform.Windows:
pythonDownloadUrl = constants.pythonWindowsInstallUrl;
break;
case utils.Platform.Mac:
pythonDownloadUrl = constants.pythonMacInstallUrl;
break;
default:
// Default to linux
pythonDownloadUrl = constants.pythonLinuxInstallUrl;
break;
}
let installPath: string;
if (this._usingExistingPython) {
installPath = utils.getUserHome();
} else {
installPath = this._pythonInstallationPath;
}
let pythonPackagePathLocal = path.join(installPath, packageName);
return new Promise((resolve, reject) => {
let installPath = this._pythonInstallationPath;
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgDownloadPython(platformId, pythonDownloadUrl));
fs.mkdirs(installPath, (err) => {
if (err) {
@@ -179,12 +161,13 @@ export class JupyterServerInstallation {
}
});
let pythonPackagePathLocal = path.join(installPath, packageName);
downloadRequest.pipe(fs.createWriteStream(pythonPackagePathLocal))
.on('close', async () => {
//unpack python zip/tar file
this.outputChannel.appendLine(msgPythonUnpackPending);
let pythonSourcePath = path.join(installPath, constants.pythonBundleVersion);
if (!this._usingExistingPython && await utils.exists(pythonSourcePath)) {
if (await utils.exists(pythonSourcePath)) {
try {
fs.removeSync(pythonSourcePath);
} catch (err) {
@@ -201,6 +184,8 @@ export class JupyterServerInstallation {
}
});
this.outputChannel.appendLine(msgPythonDownloadComplete);
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonDownloadComplete);
resolve();
}).catch(err => {
backgroundOperation.updateStatus(azdata.TaskStatus.InProgress, msgPythonUnpackError);
@@ -223,7 +208,7 @@ export class JupyterServerInstallation {
: path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
if (this._usingExistingPython) {
this._pythonPackageDir = path.join(utils.getUserHome(), 'offlinePackages');
this._pythonPackageDir = undefined;
} else {
this._pythonPackageDir = path.join(pythonSourcePath, 'offlinePackages');
}
@@ -438,41 +423,27 @@ export class JupyterServerInstallation {
installJupyterCommand = `"${this._pythonExecutable}" -m pip install --force-reinstall --no-index pip --find-links "${this._pythonPackageDir}" --no-warn-script-location`;
await this.executeStreamedCommand(installJupyterCommand);
fs.remove(this._pythonPackageDir, (err: Error) => {
if (err) {
this.outputChannel.appendLine(err.message);
}
});
this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete."));
} else {
return Promise.resolve();
}
}
private async installSparkMagic(doOnlineInstall: boolean): Promise<void> {
let installSparkMagic: string;
if (process.platform === constants.winPlatform || this._usingExistingPython) {
// Overwrite existing install of sparkmagic, since we use a custom version
let cmdOptions = this._usingExistingPython ? '--user --force-reinstall' : '--force-reinstall';
let sparkWheel = path.join(this._pythonPackageDir, `sparkmagic-${constants.sparkMagicVersion}-py3-none-any.whl`);
if (doOnlineInstall) {
installSparkMagic = `"${this._pythonExecutable}" -m pip install ${cmdOptions} "${sparkWheel}" --no-warn-script-location`;
} else {
installSparkMagic = `"${this._pythonExecutable}" -m pip install ${cmdOptions} --no-index "${sparkWheel}" --find-links "${this._pythonPackageDir}" --no-warn-script-location`;
}
}
if (installSparkMagic) {
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallingSpark', "Installing SparkMagic..."));
await this.executeStreamedCommand(installSparkMagic);
}
}
private async installPipDependencies(): Promise<void> {
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks..."));
let cmdOptions = this._usingExistingPython ? '--user' : '';
let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} jupyter==1.0.0 pandas==0.24.2`;
let installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} jupyter>=1.0.0 pandas>=0.24.2 sparkmagic>=0.12.9`;
await this.executeStreamedCommand(installCommand);
installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`;
installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator>=1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`;
await this.executeStreamedCommand(installCommand);
this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete."));
@@ -482,14 +453,14 @@ export class JupyterServerInstallation {
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallStart', "Installing required packages to run Notebooks..."));
let installCommand = `"${this.getCondaExePath()}" install -y jupyter==1.0.0 pandas==0.24.2`;
let installCommand = `"${this.getCondaExePath()}" install -y jupyter>=1.0.0 pandas>=0.24.2`;
if (process.platform !== constants.winPlatform) {
installCommand = `${installCommand} pykerberos==1.2.1`;
installCommand = `${installCommand} pykerberos>=1.2.1`;
}
await this.executeStreamedCommand(installCommand);
let cmdOptions = this._usingExistingPython ? '--user' : '';
installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`;
installCommand = `"${this._pythonExecutable}" -m pip install ${cmdOptions} sparkmagic>=0.12.9 prose-codeaccelerator>=1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net`;
await this.executeStreamedCommand(installCommand);
this.outputChannel.appendLine(localize('msgJupyterInstallDone', "... Jupyter installation complete."));