Move SQL 2019 extension's notebook code into Azure Data Studio (#4090)

This commit is contained in:
Cory Rivera
2019-02-20 10:55:49 -08:00
committed by GitHub
parent 2dd71cbe26
commit 70838c3e24
66 changed files with 8098 additions and 14 deletions

View File

@@ -0,0 +1,16 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as vscode from 'vscode';
export interface IServerInstance {
readonly port: string;
readonly uri: vscode.Uri;
configure(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
}

View File

@@ -0,0 +1,260 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as path from 'path';
import * as sqlops from 'sqlops';
import * as vscode from 'vscode';
import * as os from 'os';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import * as constants from '../common/constants';
import * as localizedConstants from '../common/localizedConstants';
import JupyterServerInstallation from './jupyterServerInstallation';
import { IServerInstance } from './common';
import * as utils from '../common/utils';
import { IPrompter, QuestionTypes, IQuestion } from '../prompts/question';
import { AppContext } from '../common/appContext';
import { ApiWrapper } from '../common/apiWrapper';
import { LocalJupyterServerManager } from './jupyterServerManager';
import { NotebookCompletionItemProvider } from '../intellisense/completionItemProvider';
import { JupyterNotebookProvider } from './jupyterNotebookProvider';
import { ConfigurePythonDialog } from '../dialog/configurePythonDialog';
import CodeAdapter from '../prompts/adapter';
let untitledCounter = 0;
export class JupyterController implements vscode.Disposable {
private _jupyterInstallation: Promise<JupyterServerInstallation>;
private _notebookInstances: IServerInstance[] = [];
private outputChannel: vscode.OutputChannel;
private prompter: IPrompter;
constructor(private appContext: AppContext) {
this.prompter = new CodeAdapter();
this.outputChannel = this.appContext.apiWrapper.createOutputChannel(constants.extensionOutputChannel);
}
private get apiWrapper(): ApiWrapper {
return this.appContext.apiWrapper;
}
public get extensionContext(): vscode.ExtensionContext {
return this.appContext && this.appContext.extensionContext;
}
public dispose(): void {
this.deactivate();
}
// PUBLIC METHODS //////////////////////////////////////////////////////
public async activate(): Promise<boolean> {
// Prompt for install if the python installation path is not defined
let jupyterInstaller = new JupyterServerInstallation(
this.extensionContext.extensionPath,
this.outputChannel,
this.apiWrapper);
if (JupyterServerInstallation.isPythonInstalled(this.apiWrapper)) {
this._jupyterInstallation = Promise.resolve(jupyterInstaller);
} else {
this._jupyterInstallation = new Promise(resolve => {
jupyterInstaller.onInstallComplete(err => {
if (!err) {
resolve(jupyterInstaller);
}
});
});
}
let notebookProvider = undefined;
notebookProvider = this.registerNotebookProvider();
sqlops.nb.onDidOpenNotebookDocument(notebook => {
if (!JupyterServerInstallation.isPythonInstalled(this.apiWrapper)) {
this.doConfigurePython(jupyterInstaller);
}
});
// Add command/task handlers
this.apiWrapper.registerTaskHandler(constants.jupyterOpenNotebookTask, (profile: sqlops.IConnectionProfile) => {
return this.handleOpenNotebookTask(profile);
});
this.apiWrapper.registerTaskHandler(constants.jupyterNewNotebookTask, (profile: sqlops.IConnectionProfile) => {
return this.saveProfileAndCreateNotebook(profile);
});
this.apiWrapper.registerCommand(constants.jupyterNewNotebookCommand, (explorerContext: sqlops.ObjectExplorerContext) => {
return this.saveProfileAndCreateNotebook(explorerContext ? explorerContext.connectionProfile : undefined);
});
this.apiWrapper.registerCommand(constants.jupyterAnalyzeCommand, (explorerContext: sqlops.ObjectExplorerContext) => {
return this.saveProfileAndAnalyzeNotebook(explorerContext);
});
this.apiWrapper.registerCommand(constants.jupyterReinstallDependenciesCommand, () => { return this.handleDependenciesReinstallation(); });
this.apiWrapper.registerCommand(constants.jupyterInstallPackages, () => { return this.doManagePackages(); });
this.apiWrapper.registerCommand(constants.jupyterConfigurePython, () => { return this.doConfigurePython(jupyterInstaller); });
let supportedFileFilter: vscode.DocumentFilter[] = [
{ scheme: 'file', language: '*' },
{ scheme: 'untitled', language: '*' }
];
this.extensionContext.subscriptions.push(this.apiWrapper.registerCompletionItemProvider(supportedFileFilter, new NotebookCompletionItemProvider(notebookProvider)));
return true;
}
private registerNotebookProvider(): JupyterNotebookProvider {
let notebookProvider = new JupyterNotebookProvider((documentUri: vscode.Uri) => new LocalJupyterServerManager({
documentPath: documentUri.fsPath,
jupyterInstallation: this._jupyterInstallation,
extensionContext: this.extensionContext,
apiWrapper: this.apiWrapper
}));
sqlops.nb.registerNotebookProvider(notebookProvider);
return notebookProvider;
}
private saveProfileAndCreateNotebook(profile: sqlops.IConnectionProfile): Promise<void> {
return this.handleNewNotebookTask(undefined, profile);
}
private saveProfileAndAnalyzeNotebook(oeContext: sqlops.ObjectExplorerContext): Promise<void> {
return this.handleNewNotebookTask(oeContext, oeContext.connectionProfile);
}
public deactivate(): void {
// Shutdown any open notebooks
this._notebookInstances.forEach(instance => { instance.stop(); });
}
// EVENT HANDLERS //////////////////////////////////////////////////////
public async getDefaultConnection(): Promise<sqlops.ConnectionInfo> {
return await this.apiWrapper.getCurrentConnection();
}
private async handleOpenNotebookTask(profile: sqlops.IConnectionProfile): Promise<void> {
let notebookFileTypeName = localize('notebookFileType', 'Notebooks');
let filter = {};
filter[notebookFileTypeName] = 'ipynb';
let uris = await this.apiWrapper.showOpenDialog({
filters: filter,
canSelectFiles: true,
canSelectMany: false
});
if (uris && uris.length > 0) {
let fileUri = uris[0];
// Verify this is a .ipynb file since this isn't actually filtered on Mac/Linux
if (path.extname(fileUri.fsPath) !== '.ipynb') {
// in the future might want additional supported types
this.apiWrapper.showErrorMessage(localize('unsupportedFileType', 'Only .ipynb Notebooks are supported'));
} else {
await sqlops.nb.showNotebookDocument(fileUri, {
connectionId: profile.id,
providerId: constants.jupyterNotebookProviderId,
preview: false
});
}
}
}
private async handleNewNotebookTask(oeContext?: sqlops.ObjectExplorerContext, profile?: sqlops.IConnectionProfile): Promise<void> {
// Ensure we get a unique ID for the notebook. For now we're using a different prefix to the built-in untitled files
// to handle this. We should look into improving this in the future
let untitledUri = vscode.Uri.parse(`untitled:Notebook-${untitledCounter++}`);
let editor = await sqlops.nb.showNotebookDocument(untitledUri, {
connectionId: profile.id,
providerId: constants.jupyterNotebookProviderId,
preview: false,
defaultKernel: {
name: 'pyspark3kernel',
display_name: 'PySpark3',
language: 'python'
}
});
if (oeContext && oeContext.nodeInfo && oeContext.nodeInfo.nodePath) {
// Get the file path after '/HDFS'
let hdfsPath: string = oeContext.nodeInfo.nodePath.substring(oeContext.nodeInfo.nodePath.indexOf('/HDFS') + '/HDFS'.length);
if (hdfsPath.length > 0) {
let analyzeCommand = '#' + localizedConstants.msgSampleCodeDataFrame + os.EOL + 'df = (spark.read.option(\"inferSchema\", \"true\")'
+ os.EOL + '.option(\"header\", \"true\")' + os.EOL + '.csv(\'{0}\'))' + os.EOL + 'df.show(10)';
// TODO re-enable insert into document once APIs are finalized.
// editor.document.cells[0].source = [analyzeCommand.replace('{0}', hdfsPath)];
editor.edit(editBuilder => {
editBuilder.replace(0, {
cell_type: 'code',
source: analyzeCommand.replace('{0}', hdfsPath)
});
});
}
}
}
private async handleDependenciesReinstallation(): Promise<void> {
if (await this.confirmReinstall()) {
this._jupyterInstallation = JupyterServerInstallation.getInstallation(
this.extensionContext.extensionPath,
this.outputChannel,
this.apiWrapper,
undefined,
true);
}
}
//Confirmation message dialog
private async confirmReinstall(): Promise<boolean> {
return await this.prompter.promptSingle<boolean>(<IQuestion>{
type: QuestionTypes.confirm,
message: localize('confirmReinstall', 'Are you sure you want to reinstall?'),
default: true
});
}
public doManagePackages(): void {
try {
let terminal = this.apiWrapper.createTerminalWithOptions({ cwd: this.getPythonBinDir() });
terminal.show(true);
let shellType = this.apiWrapper.getConfiguration().get('terminal.integrated.shell.windows');
terminal.sendText(this.getTextToSendToTerminal(shellType), true);
} catch (error) {
let message = utils.getErrorMessage(error);
this.apiWrapper.showErrorMessage(message);
}
}
public async doConfigurePython(jupyterInstaller: JupyterServerInstallation): Promise<void> {
try {
let pythonDialog = new ConfigurePythonDialog(this.appContext, this.outputChannel, jupyterInstaller);
await pythonDialog.showDialog();
} catch (error) {
let message = utils.getErrorMessage(error);
this.apiWrapper.showErrorMessage(message);
}
}
public getTextToSendToTerminal(shellType: any): string {
if (utils.getOSPlatform() === utils.Platform.Windows && typeof shellType === 'string') {
if (shellType.endsWith('powershell.exe')) {
return localizedConstants.msgManagePackagesPowershell;
} else if (shellType.endsWith('cmd.exe')) {
return localizedConstants.msgManagePackagesCmd;
} else {
return localizedConstants.msgManagePackagesBash;
}
} else {
return localizedConstants.msgManagePackagesBash;
}
}
private getPythonBinDir(): string {
return JupyterServerInstallation.getPythonBinPath(this.apiWrapper);
}
public get jupyterInstallation() {
return this._jupyterInstallation;
}
}

View File

@@ -0,0 +1,172 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import { Kernel, KernelMessage } from '@jupyterlab/services';
function toShellMessage(msgImpl: KernelMessage.IShellMessage): nb.IShellMessage {
return {
channel: msgImpl.channel,
type: msgImpl.channel,
content: msgImpl.content,
header: msgImpl.header,
parent_header: msgImpl.parent_header,
metadata: msgImpl.metadata
};
}
function toStdInMessage(msgImpl: KernelMessage.IStdinMessage): nb.IStdinMessage {
return {
channel: msgImpl.channel,
type: msgImpl.channel,
content: msgImpl.content,
header: msgImpl.header,
parent_header: msgImpl.parent_header,
metadata: msgImpl.metadata
};
}
function toIOPubMessage(msgImpl: KernelMessage.IIOPubMessage): nb.IIOPubMessage {
return {
channel: msgImpl.channel,
type: msgImpl.channel,
content: msgImpl.content,
header: msgImpl.header,
parent_header: msgImpl.parent_header,
metadata: msgImpl.metadata
};
}
function toIInputReply(content: nb.IInputReply): KernelMessage.IInputReply {
return {
value: content.value
};
}
export class JupyterKernel implements nb.IKernel {
constructor(private kernelImpl: Kernel.IKernelConnection) {
}
public get id(): string {
return this.kernelImpl.id;
}
public get name(): string {
return this.kernelImpl.name;
}
public get supportsIntellisense(): boolean {
return true;
}
public get isReady(): boolean {
return this.kernelImpl.isReady;
}
public get ready(): Promise<void> {
return this.kernelImpl.ready;
}
public get info(): nb.IInfoReply {
return this.kernelImpl.info as nb.IInfoReply;
}
public async getSpec(): Promise<nb.IKernelSpec> {
let specImpl = await this.kernelImpl.getSpec();
return {
name: specImpl.name,
display_name: specImpl.display_name
};
}
requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture {
let futureImpl = this.kernelImpl.requestExecute(content as KernelMessage.IExecuteRequest, disposeOnDone);
return new JupyterFuture(futureImpl);
}
requestComplete(content: nb.ICompleteRequest): Promise<nb.ICompleteReplyMsg> {
return this.kernelImpl.requestComplete({
code: content.code,
cursor_pos: content.cursor_pos
}).then((completeMsg) => {
// Complete msg matches shell message definition, but with clearer content body
let msg: nb.ICompleteReplyMsg = toShellMessage(completeMsg);
return msg;
});
}
interrupt(): Promise<void> {
return this.kernelImpl.interrupt();
}
}
export class JupyterFuture implements nb.IFuture {
private _inProgress: boolean;
constructor(private futureImpl: Kernel.IFuture) {
this._inProgress = true;
}
public get msg(): nb.IShellMessage {
let msgImpl = this.futureImpl.msg;
return toShellMessage(msgImpl);
}
public get done(): Promise<nb.IShellMessage> {
// Convert on success, leave to throw original error on fail
return this.futureImpl.done.then((msgImpl) => {
this._inProgress = false;
return toShellMessage(msgImpl);
});
}
public get inProgress(): boolean {
return this._inProgress;
}
public set inProgress(inProg: boolean) {
this._inProgress = inProg;
}
setReplyHandler(handler: nb.MessageHandler<nb.IShellMessage>): void {
this.futureImpl.onReply = (msg) => {
let shellMsg = toShellMessage(msg);
return handler.handle(shellMsg);
};
}
setStdInHandler(handler: nb.MessageHandler<nb.IStdinMessage>): void {
this.futureImpl.onStdin = (msg) => {
let shellMsg = toStdInMessage(msg);
return handler.handle(shellMsg);
};
}
setIOPubHandler(handler: nb.MessageHandler<nb.IIOPubMessage>): void {
this.futureImpl.onIOPub = (msg) => {
let shellMsg = toIOPubMessage(msg);
return handler.handle(shellMsg);
};
}
registerMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | PromiseLike<boolean>): void {
throw new Error('Method not implemented.');
}
removeMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | PromiseLike<boolean>): void {
throw new Error('Method not implemented.');
}
sendInputReply(content: nb.IInputReply): void {
this.futureImpl.sendInputReply(toIInputReply(content));
}
dispose(): void {
this.futureImpl.dispose();
}
}

View File

@@ -0,0 +1,56 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import * as vscode from 'vscode';
import { ServerConnection, SessionManager } from '@jupyterlab/services';
import { JupyterSessionManager } from './jupyterSessionManager';
import { ApiWrapper } from '../common/apiWrapper';
import { LocalJupyterServerManager } from './jupyterServerManager';
export class JupyterNotebookManager implements nb.NotebookManager, vscode.Disposable {
protected _serverSettings: ServerConnection.ISettings;
private _sessionManager: JupyterSessionManager;
constructor(private _serverManager: LocalJupyterServerManager, sessionManager?: JupyterSessionManager, private apiWrapper: ApiWrapper = new ApiWrapper()) {
this._sessionManager = sessionManager || new JupyterSessionManager();
this._serverManager.onServerStarted(() => {
this.setServerSettings(this._serverManager.serverSettings);
});
}
public get contentManager(): nb.ContentManager {
return undefined;
}
public get sessionManager(): nb.SessionManager {
return this._sessionManager;
}
public get serverManager(): nb.ServerManager {
return this._serverManager;
}
public get serverSettings(): ServerConnection.ISettings {
return this._serverSettings;
}
public setServerSettings(settings: Partial<ServerConnection.ISettings>): void {
this._serverSettings = ServerConnection.makeSettings(settings);
this._sessionManager.setJupyterSessionManager(new SessionManager({ serverSettings: this._serverSettings }));
}
dispose() {
if (this._sessionManager) {
this._sessionManager.shutdownAll().then(() => this._sessionManager.dispose());
}
if (this._serverManager) {
this._serverManager.stopServer().then(success => undefined, error => this.apiWrapper.showErrorMessage(error));
}
}
}

View File

@@ -0,0 +1,82 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import * as constants from '../common/constants';
import { JupyterNotebookManager } from './jupyterNotebookManager';
import { LocalJupyterServerManager } from './jupyterServerManager';
export type ServerManagerFactory = (documentUri: vscode.Uri) => LocalJupyterServerManager;
export class JupyterNotebookProvider implements nb.NotebookProvider {
readonly providerId: string = constants.jupyterNotebookProviderId;
private managerTracker = new Map<string, JupyterNotebookManager>();
constructor(private createServerManager: ServerManagerFactory) {
}
public getNotebookManager(notebookUri: vscode.Uri): Thenable<nb.NotebookManager> {
if (!notebookUri) {
return Promise.reject(localize('errNotebookUriMissing', 'A notebook path is required'));
}
return this.doGetNotebookManager(notebookUri);
}
private doGetNotebookManager(notebookUri: vscode.Uri): Promise<nb.NotebookManager> {
let uriString = notebookUri.toString();
let manager = this.managerTracker.get(uriString);
if (!manager) {
let serverManager = this.createServerManager(notebookUri);
manager = new JupyterNotebookManager(serverManager);
this.managerTracker.set(uriString, manager);
}
return Promise.resolve(manager);
}
handleNotebookClosed(notebookUri: vscode.Uri): void {
if (!notebookUri) {
// As this is a notification method, will skip throwing an error here
return;
}
let uriString = notebookUri.toString();
let manager = this.managerTracker.get(uriString);
if (manager) {
this.managerTracker.delete(uriString);
manager.dispose();
}
}
public get standardKernels(): nb.IStandardKernel[] {
return [
{
"name": "Python 3",
"connectionProviderIds": []
},
{
"name": "PySpark",
"connectionProviderIds": ["HADOOP_KNOX"]
},
{
"name": "PySpark3",
"connectionProviderIds": ["HADOOP_KNOX"]
},
{
"name": "Spark | R",
"connectionProviderIds": ["HADOOP_KNOX"]
},
{
"name": "Spark | Scala",
"connectionProviderIds": ["HADOOP_KNOX"]
}
];
}
}

View File

@@ -0,0 +1,338 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as nls from 'vscode-nls';
import * as sqlops from 'sqlops';
import { ExecOptions } from 'child_process';
import * as decompress from 'decompress';
import * as request from 'request';
import { ApiWrapper } from '../common/apiWrapper';
import * as constants from '../common/constants';
import * as utils from '../common/utils';
import { OutputChannel, ConfigurationTarget, Event, EventEmitter, window } from 'vscode';
const localize = nls.loadMessageBundle();
const msgPythonInstallationProgress = localize('msgPythonInstallationProgress', 'Python installation is in progress');
const msgPythonInstallationComplete = localize('msgPythonInstallationComplete', 'Python installation is complete');
const msgPythonDownloadError = localize('msgPythonDownloadError', 'Error while downloading python setup');
const msgPythonDownloadPending = localize('msgPythonDownloadPending', 'Downloading python package');
const msgPythonUnpackPending = localize('msgPythonUnpackPending', 'Unpacking python package');
const msgPythonDirectoryError = localize('msgPythonDirectoryError', 'Error while creating python installation directory');
const msgPythonUnpackError = localize('msgPythonUnpackError', 'Error while unpacking python bundle');
const msgTaskName = localize('msgTaskName', 'Installing Notebook dependencies');
const msgInstallPkgStart = localize('msgInstallPkgStart', 'Installing Notebook dependencies, see Tasks view for more information');
const msgInstallPkgFinish = localize('msgInstallPkgFinish', 'Notebook dependencies installation is complete');
function msgDependenciesInstallationFailed(errorMessage: string): string { return localize('msgDependenciesInstallationFailed', 'Installing Notebook dependencies failed with error: {0}', errorMessage); }
function msgDownloadPython(platform: string, pythonDownloadUrl: string): string { return localize('msgDownloadPython', 'Downloading local python for platform: {0} to {1}', platform, pythonDownloadUrl); }
export default class JupyterServerInstallation {
/**
* Path to the folder where all configuration sets will be stored. Should always be:
* %extension_path%/jupyter_config
*/
public apiWrapper: ApiWrapper;
public extensionPath: string;
public pythonBinPath: string;
public outputChannel: OutputChannel;
public configRoot: string;
public pythonEnvVarPath: string;
public execOptions: ExecOptions;
private _pythonInstallationPath: string;
private _pythonExecutable: string;
// Allows dependencies to be installed even if an existing installation is already present
private _forceInstall: boolean;
private static readonly DefaultPythonLocation = path.join(utils.getUserHome(), 'azuredatastudio-python');
private _installCompleteEmitter = new EventEmitter<string>();
constructor(extensionPath: string, outputChannel: OutputChannel, apiWrapper: ApiWrapper, pythonInstallationPath?: string, forceInstall?: boolean) {
this.extensionPath = extensionPath;
this.outputChannel = outputChannel;
this.apiWrapper = apiWrapper;
this._pythonInstallationPath = pythonInstallationPath || JupyterServerInstallation.getPythonInstallPath(this.apiWrapper);
this.configRoot = path.join(this.extensionPath, constants.jupyterConfigRootFolder);
this._forceInstall = !!forceInstall;
this.configurePackagePaths();
}
public get onInstallComplete(): Event<string> {
return this._installCompleteEmitter.event;
}
public static async getInstallation(
extensionPath: string,
outputChannel: OutputChannel,
apiWrapper: ApiWrapper,
pythonInstallationPath?: string,
forceInstall?: boolean): Promise<JupyterServerInstallation> {
let installation = new JupyterServerInstallation(extensionPath, outputChannel, apiWrapper, pythonInstallationPath, forceInstall);
await installation.startInstallProcess();
return installation;
}
private async installDependencies(backgroundOperation: sqlops.BackgroundOperation): Promise<void> {
if (!fs.existsSync(this._pythonExecutable) || this._forceInstall) {
window.showInformationMessage(msgInstallPkgStart);
this.outputChannel.show(true);
this.outputChannel.appendLine(msgPythonInstallationProgress);
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonInstallationProgress);
await this.installPythonPackage(backgroundOperation);
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonInstallationComplete);
this.outputChannel.appendLine(msgPythonInstallationComplete);
// Install jupyter on Windows because local python is not bundled with jupyter unlike linux and MacOS.
await this.installJupyterProsePackage();
await this.installSparkMagic();
backgroundOperation.updateStatus(sqlops.TaskStatus.Succeeded, msgInstallPkgFinish);
window.showInformationMessage(msgInstallPkgFinish);
}
}
private installPythonPackage(backgroundOperation: sqlops.BackgroundOperation): Promise<void> {
let bundleVersion = constants.pythonBundleVersion;
let pythonVersion = constants.pythonVersion;
let packageName = 'python-#pythonversion-#platform-#bundleversion.#extension';
let platformId = utils.getOSPlatformId();
packageName = packageName.replace('#platform', platformId)
.replace('#pythonversion', pythonVersion)
.replace('#bundleversion', bundleVersion)
.replace('#extension', process.platform === constants.winPlatform ? 'zip' : 'tar.gz');
let pythonDownloadUrl = undefined;
switch (utils.getOSPlatform()) {
case utils.Platform.Windows:
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065977';
break;
case utils.Platform.Mac:
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065976';
break;
default:
// Default to linux
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065975';
break;
}
let pythonPackagePathLocal = this._pythonInstallationPath + '/' + packageName;
let self = undefined;
return new Promise((resolve, reject) => {
self = this;
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgDownloadPython(platformId, pythonDownloadUrl));
fs.mkdirs(this._pythonInstallationPath, (err) => {
if (err) {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDirectoryError);
reject(err);
}
let totalMegaBytes: number = undefined;
let receivedBytes = 0;
let printThreshold = 0.1;
request.get(pythonDownloadUrl, { timeout: 20000 })
.on('error', (downloadError) => {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
reject(downloadError);
})
.on('response', (response) => {
if (response.statusCode !== 200) {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
reject(response.statusMessage);
}
let totalBytes = parseInt(response.headers['content-length']);
totalMegaBytes = totalBytes / (1024 * 1024);
this.outputChannel.appendLine(`${msgPythonDownloadPending} (0 / ${totalMegaBytes.toFixed(2)} MB)`);
})
.on('data', (data) => {
receivedBytes += data.length;
if (totalMegaBytes) {
let receivedMegaBytes = receivedBytes / (1024 * 1024);
let percentage = receivedMegaBytes / totalMegaBytes;
if (percentage >= printThreshold) {
this.outputChannel.appendLine(`${msgPythonDownloadPending} (${receivedMegaBytes.toFixed(2)} / ${totalMegaBytes.toFixed(2)} MB)`);
printThreshold += 0.1;
}
}
})
.pipe(fs.createWriteStream(pythonPackagePathLocal))
.on('close', () => {
//unpack python zip/tar file
this.outputChannel.appendLine(msgPythonUnpackPending);
let pythonSourcePath = path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
if (fs.existsSync(pythonSourcePath)) {
try {
fs.removeSync(pythonSourcePath);
} catch (err) {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
reject(err);
}
}
decompress(pythonPackagePathLocal, self._pythonInstallationPath).then(files => {
//Delete zip/tar file
fs.unlink(pythonPackagePathLocal, (err) => {
if (err) {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
reject(err);
}
});
resolve();
}).catch(err => {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
reject(err);
});
})
.on('error', (downloadError) => {
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
reject(downloadError);
});
});
});
}
private configurePackagePaths(): void {
//Python source path up to bundle version
let pythonSourcePath = path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
// Update python paths and properties to reference user's local python.
let pythonBinPathSuffix = process.platform === constants.winPlatform ? '' : 'bin';
this._pythonExecutable = path.join(pythonSourcePath, process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
this.pythonBinPath = path.join(pythonSourcePath, pythonBinPathSuffix);
// Store paths to python libraries required to run jupyter.
this.pythonEnvVarPath = process.env.Path;
let delimiter = path.delimiter;
if (process.platform === constants.winPlatform) {
let pythonScriptsPath = path.join(pythonSourcePath, 'Scripts');
this.pythonEnvVarPath = pythonScriptsPath + delimiter + this.pythonEnvVarPath;
}
this.pythonEnvVarPath = this.pythonBinPath + delimiter + this.pythonEnvVarPath;
// Store the executable options to run child processes with env var without interfering parent env var.
let env = Object.assign({}, process.env);
env['PATH'] = this.pythonEnvVarPath;
this.execOptions = {
env: env
};
}
public async startInstallProcess(pythonInstallationPath?: string): Promise<void> {
if (pythonInstallationPath) {
this._pythonInstallationPath = pythonInstallationPath;
this.configurePackagePaths();
}
let updateConfig = () => {
let notebookConfig = this.apiWrapper.getConfiguration(constants.notebookConfigKey);
notebookConfig.update(constants.pythonPathConfigKey, this._pythonInstallationPath, ConfigurationTarget.Global);
};
if (!fs.existsSync(this._pythonExecutable) || this._forceInstall) {
this.apiWrapper.startBackgroundOperation({
displayName: msgTaskName,
description: msgTaskName,
isCancelable: false,
operation: op => {
this.installDependencies(op)
.then(() => {
this._installCompleteEmitter.fire();
updateConfig();
})
.catch(err => {
let errorMsg = msgDependenciesInstallationFailed(err);
op.updateStatus(sqlops.TaskStatus.Failed, errorMsg);
this.apiWrapper.showErrorMessage(errorMsg);
this._installCompleteEmitter.fire(errorMsg);
});
}
});
} else {
// Python executable already exists, but the path setting wasn't defined,
// so update it here
this._installCompleteEmitter.fire();
updateConfig();
}
}
private async installJupyterProsePackage(): Promise<void> {
if (process.platform === constants.winPlatform) {
let installJupyterCommand = `${this._pythonExecutable} -m pip install pandas==0.22.0 jupyter prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net --no-warn-script-location`;
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallStart', 'Installing required packages to run Notebooks...'));
await utils.executeStreamedCommand(installJupyterCommand, this.outputChannel);
this.outputChannel.appendLine(localize('msgJupyterInstallDone', '... Jupyter installation complete.'));
} else {
return Promise.resolve();
}
}
private async installSparkMagic(): Promise<void> {
if (process.platform === constants.winPlatform) {
let sparkMagicPath = path.join(this.extensionPath, 'wheels/sparkmagic-#sparkMagicVersion-py3-none-any.whl'.replace('#sparkMagicVersion', constants.sparkMagicVersion));
let installSparkMagic = `${this._pythonExecutable} -m pip install ${sparkMagicPath} --no-warn-script-location`;
this.outputChannel.show(true);
this.outputChannel.appendLine(localize('msgInstallingSpark', 'Installing SparkMagic...'));
await utils.executeStreamedCommand(installSparkMagic, this.outputChannel);
} else {
return Promise.resolve();
}
}
public get pythonExecutable(): string {
return this._pythonExecutable;
}
public static isPythonInstalled(apiWrapper: ApiWrapper): boolean {
// Don't use _pythonExecutable here, since it could be populated with a default value
let pathSetting = JupyterServerInstallation.getPythonPathSetting(apiWrapper);
if (!pathSetting) {
return false;
}
let pythonExe = path.join(
pathSetting,
constants.pythonBundleVersion,
process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
return fs.existsSync(pythonExe);
}
public static getPythonInstallPath(apiWrapper: ApiWrapper): string {
let userPath = JupyterServerInstallation.getPythonPathSetting(apiWrapper);
return userPath ? userPath : JupyterServerInstallation.DefaultPythonLocation;
}
private static getPythonPathSetting(apiWrapper: ApiWrapper): string {
let path = undefined;
if (apiWrapper) {
let notebookConfig = apiWrapper.getConfiguration(constants.notebookConfigKey);
if (notebookConfig) {
let configPythonPath = notebookConfig[constants.pythonPathConfigKey];
if (configPythonPath && fs.existsSync(configPythonPath)) {
path = configPythonPath;
}
}
}
return path;
}
public static getPythonBinPath(apiWrapper: ApiWrapper): string {
let pythonBinPathSuffix = process.platform === constants.winPlatform ? '' : 'bin';
return path.join(
JupyterServerInstallation.getPythonInstallPath(apiWrapper),
constants.pythonBundleVersion,
pythonBinPathSuffix);
}
}

View File

@@ -0,0 +1,147 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import * as vscode from 'vscode';
import * as path from 'path';
import { ServerConnection } from '@jupyterlab/services';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import { ApiWrapper } from '../common/apiWrapper';
import JupyterServerInstallation from './jupyterServerInstallation';
import * as utils from '../common/utils';
import { IServerInstance } from './common';
import { PerNotebookServerInstance, IInstanceOptions } from './serverInstance';
export interface IServerManagerOptions {
documentPath: string;
jupyterInstallation: Promise<JupyterServerInstallation>;
extensionContext: vscode.ExtensionContext;
apiWrapper?: ApiWrapper;
factory?: ServerInstanceFactory;
}
export class LocalJupyterServerManager implements nb.ServerManager, vscode.Disposable {
private _serverSettings: Partial<ServerConnection.ISettings>;
private _onServerStarted = new vscode.EventEmitter<void>();
private _instanceOptions: IInstanceOptions;
private apiWrapper: ApiWrapper;
private jupyterServer: IServerInstance;
factory: ServerInstanceFactory;
constructor(private options: IServerManagerOptions) {
this.apiWrapper = options.apiWrapper || new ApiWrapper();
this.factory = options.factory || new ServerInstanceFactory();
}
public get serverSettings(): Partial<ServerConnection.ISettings> {
return this._serverSettings;
}
public get isStarted(): boolean {
return !!this.jupyterServer;
}
public get instanceOptions(): IInstanceOptions {
return this._instanceOptions;
}
public get onServerStarted(): vscode.Event<void> {
return this._onServerStarted.event;
}
public async startServer(): Promise<void> {
try {
this.jupyterServer = await this.doStartServer();
this.options.extensionContext.subscriptions.push(this);
let partialSettings = LocalJupyterServerManager.getLocalConnectionSettings(this.jupyterServer.uri);
this._serverSettings = partialSettings;
this._onServerStarted.fire();
} catch (error) {
this.apiWrapper.showErrorMessage(localize('startServerFailed', 'Starting local Notebook server failed with error {0}', utils.getErrorMessage(error)));
throw error;
}
}
public dispose(): void {
this.stopServer().catch(err => {
let msg = utils.getErrorMessage(err);
this.apiWrapper.showErrorMessage(localize('shutdownError', 'Shutdown of Notebook server failed: {0}', msg));
});
}
public async stopServer(): Promise<void> {
if (this.jupyterServer) {
await this.jupyterServer.stop();
}
}
public static getLocalConnectionSettings(uri: vscode.Uri): Partial<ServerConnection.ISettings> {
return {
baseUrl: `${uri.scheme}://${uri.authority}`,
token: LocalJupyterServerManager.getToken(uri.query)
};
}
private static getToken(query: string): string {
if (query) {
let parts = query.split('=');
if (parts && parts.length >= 2) {
return parts[1];
}
}
return '';
}
private get documentPath(): string {
return this.options.documentPath;
}
private async doStartServer(): Promise<IServerInstance> { // We can't find or create servers until the installation is complete
let installation = await this.options.jupyterInstallation;
// Calculate the path to use as the notebook-dir for Jupyter based on the path of the uri of the
// notebook to open. This will be the workspace folder if the notebook uri is inside a workspace
// folder. Otherwise, it will be the folder that the notebook is inside. Ultimately, this means
// a new notebook server will be started for each folder a notebook is opened from.
//
// eg, opening:
// /path1/nb1.ipynb
// /path2/nb2.ipynb
// /path2/nb3.ipynb
// ... will result in 2 notebook servers being started, one for /path1/ and one for /path2/
let notebookDir = this.apiWrapper.getWorkspacePathFromUri(vscode.Uri.file(this.documentPath));
notebookDir = notebookDir || path.dirname(this.documentPath);
// TODO handle notification of start/stop status
// notebookContext.updateLoadingMessage(localizedConstants.msgJupyterStarting);
// TODO refactor server instance so it doesn't need the kernel. Likely need to reimplement this
// for notebook version
let serverInstanceOptions: IInstanceOptions = {
documentPath: this.documentPath,
notebookDirectory: notebookDir,
install: installation
};
this._instanceOptions = serverInstanceOptions;
let server = this.factory.createInstance(serverInstanceOptions);
await server.configure();
await server.start();
return server;
}
}
export class ServerInstanceFactory {
createInstance(options: IInstanceOptions): IServerInstance {
return new PerNotebookServerInstance(options);
}
}

View File

@@ -0,0 +1,338 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb, ServerInfo, connection, IConnectionProfile } from 'sqlops';
import { Session, Kernel } from '@jupyterlab/services';
import * as fs from 'fs-extra';
import * as nls from 'vscode-nls';
import { Uri } from 'vscode';
import * as path from 'path';
import * as utils from '../common/utils';
const localize = nls.loadMessageBundle();
import { JupyterKernel } from './jupyterKernel';
import { Deferred } from '../common/promise';
const configBase = {
'kernel_python_credentials': {
'url': ''
},
'kernel_scala_credentials': {
'url': ''
},
'kernel_r_credentials': {
'url': ''
},
'ignore_ssl_errors': true,
'logging_config': {
'version': 1,
'formatters': {
'magicsFormatter': {
'format': '%(asctime)s\t%(levelname)s\t%(message)s',
'datefmt': ''
}
},
'handlers': {
'magicsHandler': {
'class': 'hdijupyterutils.filehandler.MagicsFileHandler',
'formatter': 'magicsFormatter',
'home_path': ''
}
},
'loggers': {
'magicsLogger': {
'handlers': ['magicsHandler'],
'level': 'DEBUG',
'propagate': 0
}
}
}
};
const KNOX_ENDPOINT_SERVER = 'host';
const KNOX_ENDPOINT_PORT = 'knoxport';
const KNOX_ENDPOINT = 'knox';
const SQL_PROVIDER = 'MSSQL';
const USER = 'user';
const DEFAULT_CLUSTER_USER_NAME = 'root';
export class JupyterSessionManager implements nb.SessionManager {
private _ready: Deferred<void>;
private _isReady: boolean;
private _sessionManager: Session.IManager;
private static _sessions: JupyterSession[] = [];
constructor() {
this._isReady = false;
this._ready = new Deferred<void>();
}
public setJupyterSessionManager(sessionManager: Session.IManager): void {
this._sessionManager = sessionManager;
sessionManager.ready
.then(() => {
this._isReady = true;
this._ready.resolve();
}).catch((error) => {
this._isReady = false;
this._ready.reject(error);
});
}
public get isReady(): boolean {
return this._isReady;
}
public get ready(): Promise<void> {
return this._ready.promise;
}
public get specs(): nb.IAllKernels | undefined {
if (!this._isReady) {
return undefined;
}
let specs = this._sessionManager.specs;
if (!specs) {
return undefined;
}
let kernels: nb.IKernelSpec[] = Object.keys(specs.kernelspecs).map(k => {
let value = specs.kernelspecs[k];
let kernel: nb.IKernelSpec = {
name: k,
display_name: value.display_name ? value.display_name : k
};
// TODO add more info to kernels
return kernel;
});
let allKernels: nb.IAllKernels = {
defaultKernel: specs.default,
kernels: kernels
};
return allKernels;
}
public async startNew(options: nb.ISessionOptions): Promise<nb.ISession> {
if (!this._isReady) {
// no-op
return Promise.reject(new Error(localize('errorStartBeforeReady', 'Cannot start a session, the manager is not yet initialized')));
}
let sessionImpl = await this._sessionManager.startNew(options);
let jupyterSession = new JupyterSession(sessionImpl);
let index = JupyterSessionManager._sessions.findIndex(session => session.path === options.path);
if (index > -1) {
JupyterSessionManager._sessions.splice(index);
}
JupyterSessionManager._sessions.push(jupyterSession);
return jupyterSession;
}
public listRunning(): JupyterSession[] {
return JupyterSessionManager._sessions;
}
public shutdown(id: string): Promise<void> {
if (!this._isReady) {
// no-op
return Promise.resolve();
}
let index = JupyterSessionManager._sessions.findIndex(session => session.id === id);
if (index > -1) {
JupyterSessionManager._sessions.splice(index);
}
if (this._sessionManager && !this._sessionManager.isDisposed) {
return this._sessionManager.shutdown(id);
}
}
public shutdownAll(): Promise<void> {
return this._sessionManager.shutdownAll();
}
public dispose(): void {
this._sessionManager.dispose();
}
}
export class JupyterSession implements nb.ISession {
private _kernel: nb.IKernel;
constructor(private sessionImpl: Session.ISession) {
}
public get canChangeKernels(): boolean {
return true;
}
public get id(): string {
return this.sessionImpl.id;
}
public get path(): string {
return this.sessionImpl.path;
}
public get name(): string {
return this.sessionImpl.name;
}
public get type(): string {
return this.sessionImpl.type;
}
public get status(): nb.KernelStatus {
return this.sessionImpl.status;
}
public get kernel(): nb.IKernel {
if (!this._kernel) {
let kernelImpl = this.sessionImpl.kernel;
if (kernelImpl) {
this._kernel = new JupyterKernel(kernelImpl);
}
}
return this._kernel;
}
public async changeKernel(kernelInfo: nb.IKernelSpec): Promise<nb.IKernel> {
// For now, Jupyter implementation handles disposal etc. so we can just
// null out our kernel and let the changeKernel call handle this
this._kernel = undefined;
// For now, just using name. It's unclear how we'd know the ID
let options: Partial<Kernel.IModel> = {
name: kernelInfo.name
};
return this.sessionImpl.changeKernel(options).then((kernelImpl) => {
this._kernel = new JupyterKernel(kernelImpl);
return this._kernel;
});
}
public async configureKernel(): Promise<void> {
let sparkmagicConfDir = path.join(utils.getUserHome(), '.sparkmagic');
await utils.mkDir(sparkmagicConfDir);
// Default to localhost in config file.
let creds: ICredentials = {
'url': 'http://localhost:8088'
};
let config: ISparkMagicConfig = Object.assign({}, configBase);
this.updateConfig(config, creds, sparkmagicConfDir);
let configFilePath = path.join(sparkmagicConfDir, 'config.json');
await fs.writeFile(configFilePath, JSON.stringify(config));
}
public async configureConnection(connection: IConnectionProfile): Promise<void> {
if (connection && connection.providerName && this.isSparkKernel(this.sessionImpl.kernel.name)) {
// TODO may need to reenable a way to get the credential
// await this._connection.getCredential();
// %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options,
// such as user/profile/host name/auth type
//Update server info with bigdata endpoint - Unified Connection
if (connection.providerName === SQL_PROVIDER) {
let clusterEndpoint: IEndpoint = await this.getClusterEndpoint(connection.id, KNOX_ENDPOINT);
if (!clusterEndpoint) {
let kernelDisplayName: string = await this.getKernelDisplayName();
return Promise.reject(new Error(localize('connectionNotValid', 'Spark kernels require a connection to a SQL Server big data cluster master instance.')));
}
connection.options[KNOX_ENDPOINT_SERVER] = clusterEndpoint.ipAddress;
connection.options[KNOX_ENDPOINT_PORT] = clusterEndpoint.port;
connection.options[USER] = DEFAULT_CLUSTER_USER_NAME;
}
else {
connection.options[KNOX_ENDPOINT_PORT] = this.getKnoxPortOrDefault(connection);
}
this.setHostAndPort(':', connection);
this.setHostAndPort(',', connection);
let server = Uri.parse(utils.getLivyUrl(connection.options[KNOX_ENDPOINT_SERVER], connection.options[KNOX_ENDPOINT_PORT])).toString();
let doNotCallChangeEndpointParams =
`%_do_not_call_change_endpoint --username=${connection.options[USER]} --password=${connection.options['password']} --server=${server} --auth=Basic_Access`;
let future = this.sessionImpl.kernel.requestExecute({
code: doNotCallChangeEndpointParams
}, true);
await future.done;
}
}
private async getKernelDisplayName(): Promise<string> {
let spec = await this.kernel.getSpec();
return spec.display_name;
}
private isSparkKernel(kernelName: string): boolean {
return kernelName && kernelName.toLowerCase().indexOf('spark') > -1;
}
private setHostAndPort(delimeter: string, connection: IConnectionProfile): void {
let originalHost = connection.options[KNOX_ENDPOINT_SERVER];
if (!originalHost) {
return;
}
let index = originalHost.indexOf(delimeter);
if (index > -1) {
connection.options[KNOX_ENDPOINT_SERVER] = originalHost.slice(0, index);
connection.options[KNOX_ENDPOINT_PORT] = originalHost.slice(index + 1);
}
}
private updateConfig(config: ISparkMagicConfig, creds: ICredentials, homePath: string): void {
config.kernel_python_credentials = creds;
config.kernel_scala_credentials = creds;
config.kernel_r_credentials = creds;
config.logging_config.handlers.magicsHandler.home_path = homePath;
}
private getKnoxPortOrDefault(connectionProfile: IConnectionProfile): string {
let port = connectionProfile.options[KNOX_ENDPOINT_PORT];
if (!port) {
port = '30443';
}
return port;
}
private async getClusterEndpoint(profileId: string, serviceName: string): Promise<IEndpoint> {
let serverInfo: ServerInfo = await connection.getServerInfo(profileId);
if (!serverInfo || !serverInfo.options) {
return undefined;
}
let endpoints: IEndpoint[] = serverInfo.options['clusterEndpoints'];
if (!endpoints || endpoints.length === 0) {
return undefined;
}
return endpoints.find(ep => ep.serviceName.toLowerCase() === serviceName.toLowerCase());
}
}
interface ICredentials {
'url': string;
}
interface IEndpoint {
serviceName: string;
ipAddress: string;
port: number;
}
interface ISparkMagicConfig {
kernel_python_credentials: ICredentials;
kernel_scala_credentials: ICredentials;
kernel_r_credentials: ICredentials;
ignore_ssl_errors?: boolean;
logging_config: {
handlers: {
magicsHandler: {
home_path: string;
class?: string;
formatter?: string
}
}
};
}

View File

@@ -0,0 +1,76 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as fs from 'fs-extra';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import * as constants from '../common/constants';
export enum SettingType {
String,
Number,
Boolean,
Set
}
export class ISetting {
key: string;
value: string | number | boolean;
type: SettingType;
}
export class JupyterSettingWriter {
private settings: ISetting[] = [];
constructor(private baseFile: string) {
}
public addSetting(setting: ISetting): void {
this.settings.push(setting);
}
public async writeSettings(targetFile: string): Promise<void> {
let settings = await this.printSettings();
await fs.writeFile(targetFile, settings);
}
public async printSettings(): Promise<string> {
let content = '';
let newLine = process.platform === constants.winPlatform ? '\r\n' : '\n';
if (this.baseFile) {
let sourceContents = await fs.readFile(this.baseFile);
content += sourceContents.toString();
}
for (let setting of this.settings) {
content += newLine;
content += this.printSetting(setting);
}
return content;
}
private printSetting(setting: ISetting): string {
let value: string;
switch (setting.type) {
case SettingType.Boolean:
value = setting.value ? 'True' : 'False';
break;
case SettingType.String:
value = `'${setting.value}'`;
break;
case SettingType.Number:
value = `${setting.value}`;
break;
case SettingType.Set:
value = `set([${setting.value}])`;
break;
default:
throw new Error(localize('UnexpectedSettingType', 'Unexpected setting type {0}', setting.type));
}
return `c.${setting.key} = ${value}`;
}
}

View File

@@ -0,0 +1,43 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import * as vscode from 'vscode';
import { Contents } from '@jupyterlab/services';
export class RemoteContentManager implements nb.ContentManager {
constructor(private contents: Contents.IManager) {
}
public getNotebookContents(notebookUri: vscode.Uri): Thenable<nb.INotebookContents> {
return this.getNotebookContentsAsync(notebookUri.fsPath);
}
private async getNotebookContentsAsync(path: string): Promise<nb.INotebookContents> {
if (!path) {
return undefined;
}
// Note: intentionally letting caller handle exceptions
let contentsModel = await this.contents.get(path);
if (!contentsModel) {
return undefined;
}
return <nb.INotebookContents>contentsModel.content;
}
public async save(notebookUri: vscode.Uri, notebook: nb.INotebookContents): Promise<nb.INotebookContents> {
let path = notebookUri.fsPath;
await this.contents.save(path, {
path: path,
content: notebook,
type: 'notebook',
format: 'json'
});
return notebook;
}
}

View File

@@ -0,0 +1,399 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as UUID from 'vscode-languageclient/lib/utils/uuid';
import * as vscode from 'vscode';
import * as path from 'path';
import * as fs from 'fs-extra';
import * as os from 'os';
import { spawn, ExecOptions, SpawnOptions, ChildProcess } from 'child_process';
import * as nls from 'vscode-nls';
const localize = nls.loadMessageBundle();
import { IServerInstance } from './common';
import JupyterServerInstallation from './jupyterServerInstallation';
import * as utils from '../common/utils';
import * as constants from '../common/constants';
import * as notebookUtils from '../common/notebookUtils';
import * as ports from '../common/ports';
const NotebookConfigFilename = 'jupyter_notebook_config.py';
const CustomJsFilename = 'custom.js';
const defaultPort = 8888;
const JupyterStartedMessage = 'The Jupyter Notebook is running';
type MessageListener = (data: string | Buffer) => void;
type ErrorListener = (err: any) => void;
export interface IInstanceOptions {
/**
* The path to the initial document we want to start this server for
*/
documentPath: string;
/**
* Base install information needed in order to start the server instance
*/
install: JupyterServerInstallation;
/**
* Optional start directory for the notebook server. If none is set, will use a
* path relative to the initial document
*/
notebookDirectory?: string;
}
/**
* Helper class to enable testing without calling into file system or
* commandline shell APIs
*/
export class ServerInstanceUtils {
public mkDir(dirPath: string, outputChannel?: vscode.OutputChannel): Promise<void> {
return utils.mkDir(dirPath, outputChannel);
}
public removeDir(dirPath: string): Promise<void> {
return fs.remove(dirPath);
}
public pathExists(dirPath: string): Promise<boolean> {
return fs.pathExists(dirPath);
}
public copy(src: string, dest: string): Promise<void> {
return fs.copy(src, dest);
}
public existsSync(dirPath: string): boolean {
return fs.existsSync(dirPath);
}
public generateUuid(): string {
return UUID.generateUuid();
}
public executeBufferedCommand(cmd: string, options: ExecOptions, outputChannel?: vscode.OutputChannel): Thenable<string> {
return utils.executeBufferedCommand(cmd, options, outputChannel);
}
public spawn(command: string, args?: ReadonlyArray<string>, options?: SpawnOptions): ChildProcess {
return spawn(command, args, options);
}
public checkProcessDied(childProcess: ChildProcess): void {
if (!childProcess) {
return;
}
// Wait 10 seconds and then force kill. Jupyter stop is slow so this seems a reasonable time limit
setTimeout(() => {
// Test if the process is still alive. Throws an exception if not
try {
process.kill(childProcess.pid, <any>0);
} catch (error) {
// All is fine.
}
}, 10000);
}
}
export class PerNotebookServerInstance implements IServerInstance {
/**
* Root of the jupyter directory structure. Config and data roots will be
* under this, in order to simplify deletion of folders on stop of the instance
*/
private baseDir: string;
/**
* Path to configuration folder for this instance. Typically:
* %extension_path%/jupyter_config/%server%_config
*/
private instanceConfigRoot: string;
/**
* Path to data folder for this instance. Typically:
* %extension_path%/jupyter_config/%server%_data
*/
private instanceDataRoot: string;
private _systemJupyterDir: string;
private _port: string;
private _uri: vscode.Uri;
private _isStarted: boolean = false;
private utils: ServerInstanceUtils;
private childProcess: ChildProcess;
private errorHandler: ErrorHandler = new ErrorHandler();
constructor(private options: IInstanceOptions, fsUtils?: ServerInstanceUtils) {
this.utils = fsUtils || new ServerInstanceUtils();
}
public get isStarted(): boolean {
return this._isStarted;
}
public get port(): string {
return this._port;
}
public get uri(): vscode.Uri {
return this._uri;
}
public async configure(): Promise<void> {
await this.configureJupyter();
}
public async start(): Promise<void> {
await this.startInternal();
}
public async stop(): Promise<void> {
try {
if (this.baseDir) {
let exists = await this.utils.pathExists(this.baseDir);
if (exists) {
await this.utils.removeDir(this.baseDir);
}
}
if (this.isStarted) {
let install = this.options.install;
let stopCommand = `${install.pythonExecutable} -m jupyter notebook stop ${this._port}`;
await this.utils.executeBufferedCommand(stopCommand, install.execOptions, install.outputChannel);
this._isStarted = false;
this.utils.checkProcessDied(this.childProcess);
this.handleConnectionClosed();
}
} catch (error) {
// For now, we don't care as this is non-critical
this.notify(this.options.install, localize('serverStopError', 'Error stopping Notebook Server: {0}', utils.getErrorMessage(error)));
}
}
private async configureJupyter(): Promise<void> {
await this.createInstanceFolders();
let resourcesFolder = path.join(this.options.install.extensionPath, 'resources', constants.jupyterConfigRootFolder);
await this.copyInstanceConfig(resourcesFolder);
await this.CopyCustomJs(resourcesFolder);
await this.copyKernelsToSystemJupyterDirs();
}
private async createInstanceFolders(): Promise<void> {
this.baseDir = path.join(this.options.install.configRoot, 'instances', `${this.utils.generateUuid()}`);
this.instanceConfigRoot = path.join(this.baseDir, 'config');
this.instanceDataRoot = path.join(this.baseDir, 'data');
await this.utils.mkDir(this.baseDir, this.options.install.outputChannel);
await this.utils.mkDir(this.instanceConfigRoot, this.options.install.outputChannel);
await this.utils.mkDir(this.instanceDataRoot, this.options.install.outputChannel);
}
private async copyInstanceConfig(resourcesFolder: string): Promise<void> {
let configSource = path.join(resourcesFolder, NotebookConfigFilename);
let configDest = path.join(this.instanceConfigRoot, NotebookConfigFilename);
await this.utils.copy(configSource, configDest);
}
private async CopyCustomJs(resourcesFolder: string): Promise<void> {
let customPath = path.join(this.instanceConfigRoot, 'custom');
await this.utils.mkDir(customPath, this.options.install.outputChannel);
let customSource = path.join(resourcesFolder, CustomJsFilename);
let customDest = path.join(customPath, CustomJsFilename);
await this.utils.copy(customSource, customDest);
}
private async copyKernelsToSystemJupyterDirs(): Promise<void> {
let kernelsExtensionSource = path.join(this.options.install.extensionPath, 'kernels');
this._systemJupyterDir = this.getSystemJupyterKernelDir();
if (!this.utils.existsSync(this._systemJupyterDir)) {
await this.utils.mkDir(this._systemJupyterDir, this.options.install.outputChannel);
}
await this.utils.copy(kernelsExtensionSource, this._systemJupyterDir);
}
private getSystemJupyterKernelDir(): string {
switch (process.platform) {
case 'win32':
let appDataWindows = process.env['APPDATA'];
return appDataWindows + '\\jupyter\\kernels';
case 'darwin':
return path.resolve(os.homedir(), 'Library/Jupyter/kernels');
default:
return path.resolve(os.homedir(), '.local/share/jupyter/kernels');
}
}
/**
* Starts a Jupyter instance using the provided a start command. Server is determined to have
* started when the log message with URL to connect to is emitted.
* @returns {Promise<void>}
*/
protected async startInternal(): Promise<void> {
if (this.isStarted) {
return;
}
let notebookDirectory = this.getNotebookDirectory();
// Find a port in a given range. If run into trouble, got up 100 in range and search inside a larger range
let port = await ports.strictFindFreePort(new ports.StrictPortFindOptions(defaultPort, defaultPort + 100, defaultPort + 1000));
let token = await notebookUtils.getRandomToken();
this._uri = vscode.Uri.parse(`http://localhost:${port}/?token=${token}`);
this._port = port.toString();
let startCommand = `${this.options.install.pythonExecutable} -m jupyter notebook --no-browser --notebook-dir "${notebookDirectory}" --port=${port} --NotebookApp.token=${token}`;
this.notifyStarting(this.options.install, startCommand);
// Execute the command
await this.executeStartCommand(startCommand);
}
private executeStartCommand(startCommand: string): Promise<void> {
return new Promise<void>((resolve, reject) => {
let install = this.options.install;
this.childProcess = this.spawnJupyterProcess(install, startCommand);
// Add listeners for the process exiting prematurely
let onErrorBeforeStartup = (err) => reject(err);
let onExitBeforeStart = (err) => {
if (!this.isStarted) {
reject(localize('notebookStartProcessExitPremature', 'Notebook process exited prematurely with error: {0}', err));
}
};
this.childProcess.on('error', onErrorBeforeStartup);
this.childProcess.on('exit', onExitBeforeStart);
// Add listener for the process to emit its web address
let handleStdout = (data: string | Buffer) => { install.outputChannel.appendLine(data.toString()); };
let handleStdErr = (data: string | Buffer) => {
// For some reason, URL info is sent on StdErr
let [url, port] = this.matchUrlAndPort(data);
if (url) {
// For now, will verify port matches
if (url.authority !== this._uri.authority
|| url.query !== this._uri.query) {
this._uri = url;
this._port = port;
}
this.notifyStarted(install, url.toString());
this._isStarted = true;
this.updateListeners(handleStdout, handleStdErr, onErrorBeforeStartup, onExitBeforeStart);
resolve();
}
};
this.childProcess.stdout.on('data', handleStdout);
this.childProcess.stderr.on('data', handleStdErr);
});
}
private updateListeners(handleStdout: MessageListener, handleStdErr: MessageListener, onErrorBeforeStartup: ErrorListener, onExitBeforeStart: ErrorListener): void {
this.childProcess.stdout.removeListener('data', handleStdout);
this.childProcess.stderr.removeListener('data', handleStdErr);
this.childProcess.removeListener('error', onErrorBeforeStartup);
this.childProcess.removeListener('exit', onExitBeforeStart);
this.childProcess.addListener('error', this.handleConnectionError);
this.childProcess.addListener('exit', this.handleConnectionClosed);
// TODO #897 covers serializing stdout and stderr to a location where we can read from so that user can see if they run into trouble
}
private handleConnectionError(error: Error): void {
let action = this.errorHandler.handleError(error);
if (action === ErrorAction.Shutdown) {
this.notify(this.options.install, localize('jupyterError', 'Error sent from Jupyter: {0}', utils.getErrorMessage(error)));
this.stop();
}
}
private handleConnectionClosed(): void {
this.childProcess = undefined;
this._isStarted = false;
}
getNotebookDirectory(): string {
if (this.options.notebookDirectory) {
if (this.options.notebookDirectory.endsWith('\\')) {
return this.options.notebookDirectory.substr(0, this.options.notebookDirectory.length - 1) + '/';
}
return this.options.notebookDirectory;
}
return path.dirname(this.options.documentPath);
}
private matchUrlAndPort(data: string | Buffer): [vscode.Uri, string] {
// regex: Looks for the successful startup log message like:
// [C 12:08:51.947 NotebookApp]
//
// Copy/paste this URL into your browser when you connect for the first time,
// to login with a token:
// http://localhost:8888/?token=f5ee846e9bd61c3a8d835ecd9b965591511a331417b997b7
let dataString = data.toString();
let urlMatch = dataString.match(/\[C[\s\S]+ {8}(.+:(\d+)\/.*)$/m);
if (urlMatch) {
// Legacy case: manually parse token info if no token/port were passed
return [vscode.Uri.parse(urlMatch[1]), urlMatch[2]];
} else if (this._uri && dataString.indexOf(JupyterStartedMessage) > -1) {
// Default case: detect the notebook started message, indicating our preferred port and token were used
return [this._uri, this._port];
}
return [undefined, undefined];
}
private notifyStarted(install: JupyterServerInstallation, jupyterUri: string): void {
install.outputChannel.appendLine(localize('jupyterOutputMsgStartSuccessful', '... Jupyter is running at {0}', jupyterUri));
}
private notify(install: JupyterServerInstallation, message: string): void {
install.outputChannel.appendLine(message);
}
private notifyStarting(install: JupyterServerInstallation, startCommand: string): void {
install.outputChannel.appendLine(localize('jupyterOutputMsgStart', '... Starting Notebook server'));
install.outputChannel.appendLine(` > ${startCommand}`);
}
private spawnJupyterProcess(install: JupyterServerInstallation, startCommand: string): ChildProcess {
// Specify the global environment variables
let env = this.getEnvWithConfigPaths();
// Setting the PATH variable here for the jupyter command. Apparently setting it above will cause the
// notebook process to die even though we don't override it with the for loop logic above.
delete env['Path'];
env['PATH'] = install.pythonEnvVarPath;
// 'MSHOST_TELEMETRY_ENABLED' and 'MSHOST_ENVIRONMENT' environment variables are set
// for telemetry purposes used by PROSE in the process where the Jupyter kernel runs
if (vscode.workspace.getConfiguration('telemetry').get<boolean>('enableTelemetry', true)) {
env['MSHOST_TELEMETRY_ENABLED'] = true;
} else {
env['MSHOST_TELEMETRY_ENABLED'] = false;
}
env['MSHOST_ENVIRONMENT'] = 'ADSClient-' + vscode.version;
// Start the notebook process
let options = {
shell: true,
env: env
};
let childProcess = this.utils.spawn(startCommand, [], options);
return childProcess;
}
private getEnvWithConfigPaths(): any {
let env = Object.assign({}, process.env);
env['JUPYTER_CONFIG_DIR'] = this.instanceConfigRoot;
env['JUPYTER_PATH'] = this.instanceDataRoot;
return env;
}
}
class ErrorHandler {
private numErrors: number = 0;
public handleError(error: Error): ErrorAction {
this.numErrors++;
return this.numErrors > 3 ? ErrorAction.Shutdown : ErrorAction.Continue;
}
}
enum ErrorAction {
Continue = 1,
Shutdown = 2
}