mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-02-16 10:58:30 -05:00
Move SQL 2019 extension's notebook code into Azure Data Studio (#4090)
This commit is contained in:
16
extensions/notebook/src/jupyter/common.ts
Normal file
16
extensions/notebook/src/jupyter/common.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as vscode from 'vscode';
|
||||
|
||||
export interface IServerInstance {
|
||||
readonly port: string;
|
||||
readonly uri: vscode.Uri;
|
||||
configure(): Promise<void>;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
}
|
||||
260
extensions/notebook/src/jupyter/jupyterController.ts
Normal file
260
extensions/notebook/src/jupyter/jupyterController.ts
Normal file
@@ -0,0 +1,260 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as path from 'path';
|
||||
import * as sqlops from 'sqlops';
|
||||
import * as vscode from 'vscode';
|
||||
import * as os from 'os';
|
||||
import * as nls from 'vscode-nls';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import * as constants from '../common/constants';
|
||||
import * as localizedConstants from '../common/localizedConstants';
|
||||
import JupyterServerInstallation from './jupyterServerInstallation';
|
||||
import { IServerInstance } from './common';
|
||||
import * as utils from '../common/utils';
|
||||
import { IPrompter, QuestionTypes, IQuestion } from '../prompts/question';
|
||||
|
||||
import { AppContext } from '../common/appContext';
|
||||
import { ApiWrapper } from '../common/apiWrapper';
|
||||
import { LocalJupyterServerManager } from './jupyterServerManager';
|
||||
import { NotebookCompletionItemProvider } from '../intellisense/completionItemProvider';
|
||||
import { JupyterNotebookProvider } from './jupyterNotebookProvider';
|
||||
import { ConfigurePythonDialog } from '../dialog/configurePythonDialog';
|
||||
import CodeAdapter from '../prompts/adapter';
|
||||
|
||||
let untitledCounter = 0;
|
||||
|
||||
export class JupyterController implements vscode.Disposable {
|
||||
private _jupyterInstallation: Promise<JupyterServerInstallation>;
|
||||
private _notebookInstances: IServerInstance[] = [];
|
||||
|
||||
private outputChannel: vscode.OutputChannel;
|
||||
private prompter: IPrompter;
|
||||
|
||||
constructor(private appContext: AppContext) {
|
||||
this.prompter = new CodeAdapter();
|
||||
this.outputChannel = this.appContext.apiWrapper.createOutputChannel(constants.extensionOutputChannel);
|
||||
}
|
||||
|
||||
private get apiWrapper(): ApiWrapper {
|
||||
return this.appContext.apiWrapper;
|
||||
}
|
||||
|
||||
public get extensionContext(): vscode.ExtensionContext {
|
||||
return this.appContext && this.appContext.extensionContext;
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this.deactivate();
|
||||
}
|
||||
|
||||
// PUBLIC METHODS //////////////////////////////////////////////////////
|
||||
public async activate(): Promise<boolean> {
|
||||
// Prompt for install if the python installation path is not defined
|
||||
let jupyterInstaller = new JupyterServerInstallation(
|
||||
this.extensionContext.extensionPath,
|
||||
this.outputChannel,
|
||||
this.apiWrapper);
|
||||
if (JupyterServerInstallation.isPythonInstalled(this.apiWrapper)) {
|
||||
this._jupyterInstallation = Promise.resolve(jupyterInstaller);
|
||||
} else {
|
||||
this._jupyterInstallation = new Promise(resolve => {
|
||||
jupyterInstaller.onInstallComplete(err => {
|
||||
if (!err) {
|
||||
resolve(jupyterInstaller);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
let notebookProvider = undefined;
|
||||
|
||||
notebookProvider = this.registerNotebookProvider();
|
||||
sqlops.nb.onDidOpenNotebookDocument(notebook => {
|
||||
if (!JupyterServerInstallation.isPythonInstalled(this.apiWrapper)) {
|
||||
this.doConfigurePython(jupyterInstaller);
|
||||
}
|
||||
});
|
||||
// Add command/task handlers
|
||||
this.apiWrapper.registerTaskHandler(constants.jupyterOpenNotebookTask, (profile: sqlops.IConnectionProfile) => {
|
||||
return this.handleOpenNotebookTask(profile);
|
||||
});
|
||||
this.apiWrapper.registerTaskHandler(constants.jupyterNewNotebookTask, (profile: sqlops.IConnectionProfile) => {
|
||||
return this.saveProfileAndCreateNotebook(profile);
|
||||
});
|
||||
this.apiWrapper.registerCommand(constants.jupyterNewNotebookCommand, (explorerContext: sqlops.ObjectExplorerContext) => {
|
||||
return this.saveProfileAndCreateNotebook(explorerContext ? explorerContext.connectionProfile : undefined);
|
||||
});
|
||||
this.apiWrapper.registerCommand(constants.jupyterAnalyzeCommand, (explorerContext: sqlops.ObjectExplorerContext) => {
|
||||
return this.saveProfileAndAnalyzeNotebook(explorerContext);
|
||||
});
|
||||
|
||||
this.apiWrapper.registerCommand(constants.jupyterReinstallDependenciesCommand, () => { return this.handleDependenciesReinstallation(); });
|
||||
this.apiWrapper.registerCommand(constants.jupyterInstallPackages, () => { return this.doManagePackages(); });
|
||||
this.apiWrapper.registerCommand(constants.jupyterConfigurePython, () => { return this.doConfigurePython(jupyterInstaller); });
|
||||
|
||||
let supportedFileFilter: vscode.DocumentFilter[] = [
|
||||
{ scheme: 'file', language: '*' },
|
||||
{ scheme: 'untitled', language: '*' }
|
||||
];
|
||||
this.extensionContext.subscriptions.push(this.apiWrapper.registerCompletionItemProvider(supportedFileFilter, new NotebookCompletionItemProvider(notebookProvider)));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private registerNotebookProvider(): JupyterNotebookProvider {
|
||||
let notebookProvider = new JupyterNotebookProvider((documentUri: vscode.Uri) => new LocalJupyterServerManager({
|
||||
documentPath: documentUri.fsPath,
|
||||
jupyterInstallation: this._jupyterInstallation,
|
||||
extensionContext: this.extensionContext,
|
||||
apiWrapper: this.apiWrapper
|
||||
}));
|
||||
sqlops.nb.registerNotebookProvider(notebookProvider);
|
||||
return notebookProvider;
|
||||
}
|
||||
|
||||
private saveProfileAndCreateNotebook(profile: sqlops.IConnectionProfile): Promise<void> {
|
||||
return this.handleNewNotebookTask(undefined, profile);
|
||||
}
|
||||
|
||||
private saveProfileAndAnalyzeNotebook(oeContext: sqlops.ObjectExplorerContext): Promise<void> {
|
||||
return this.handleNewNotebookTask(oeContext, oeContext.connectionProfile);
|
||||
}
|
||||
|
||||
public deactivate(): void {
|
||||
// Shutdown any open notebooks
|
||||
this._notebookInstances.forEach(instance => { instance.stop(); });
|
||||
}
|
||||
|
||||
// EVENT HANDLERS //////////////////////////////////////////////////////
|
||||
public async getDefaultConnection(): Promise<sqlops.ConnectionInfo> {
|
||||
return await this.apiWrapper.getCurrentConnection();
|
||||
}
|
||||
|
||||
private async handleOpenNotebookTask(profile: sqlops.IConnectionProfile): Promise<void> {
|
||||
let notebookFileTypeName = localize('notebookFileType', 'Notebooks');
|
||||
let filter = {};
|
||||
filter[notebookFileTypeName] = 'ipynb';
|
||||
let uris = await this.apiWrapper.showOpenDialog({
|
||||
filters: filter,
|
||||
canSelectFiles: true,
|
||||
canSelectMany: false
|
||||
});
|
||||
if (uris && uris.length > 0) {
|
||||
let fileUri = uris[0];
|
||||
// Verify this is a .ipynb file since this isn't actually filtered on Mac/Linux
|
||||
if (path.extname(fileUri.fsPath) !== '.ipynb') {
|
||||
// in the future might want additional supported types
|
||||
this.apiWrapper.showErrorMessage(localize('unsupportedFileType', 'Only .ipynb Notebooks are supported'));
|
||||
} else {
|
||||
await sqlops.nb.showNotebookDocument(fileUri, {
|
||||
connectionId: profile.id,
|
||||
providerId: constants.jupyterNotebookProviderId,
|
||||
preview: false
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async handleNewNotebookTask(oeContext?: sqlops.ObjectExplorerContext, profile?: sqlops.IConnectionProfile): Promise<void> {
|
||||
// Ensure we get a unique ID for the notebook. For now we're using a different prefix to the built-in untitled files
|
||||
// to handle this. We should look into improving this in the future
|
||||
let untitledUri = vscode.Uri.parse(`untitled:Notebook-${untitledCounter++}`);
|
||||
let editor = await sqlops.nb.showNotebookDocument(untitledUri, {
|
||||
connectionId: profile.id,
|
||||
providerId: constants.jupyterNotebookProviderId,
|
||||
preview: false,
|
||||
defaultKernel: {
|
||||
name: 'pyspark3kernel',
|
||||
display_name: 'PySpark3',
|
||||
language: 'python'
|
||||
}
|
||||
});
|
||||
if (oeContext && oeContext.nodeInfo && oeContext.nodeInfo.nodePath) {
|
||||
// Get the file path after '/HDFS'
|
||||
let hdfsPath: string = oeContext.nodeInfo.nodePath.substring(oeContext.nodeInfo.nodePath.indexOf('/HDFS') + '/HDFS'.length);
|
||||
if (hdfsPath.length > 0) {
|
||||
let analyzeCommand = '#' + localizedConstants.msgSampleCodeDataFrame + os.EOL + 'df = (spark.read.option(\"inferSchema\", \"true\")'
|
||||
+ os.EOL + '.option(\"header\", \"true\")' + os.EOL + '.csv(\'{0}\'))' + os.EOL + 'df.show(10)';
|
||||
// TODO re-enable insert into document once APIs are finalized.
|
||||
// editor.document.cells[0].source = [analyzeCommand.replace('{0}', hdfsPath)];
|
||||
editor.edit(editBuilder => {
|
||||
editBuilder.replace(0, {
|
||||
cell_type: 'code',
|
||||
source: analyzeCommand.replace('{0}', hdfsPath)
|
||||
});
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async handleDependenciesReinstallation(): Promise<void> {
|
||||
if (await this.confirmReinstall()) {
|
||||
this._jupyterInstallation = JupyterServerInstallation.getInstallation(
|
||||
this.extensionContext.extensionPath,
|
||||
this.outputChannel,
|
||||
this.apiWrapper,
|
||||
undefined,
|
||||
true);
|
||||
}
|
||||
}
|
||||
|
||||
//Confirmation message dialog
|
||||
private async confirmReinstall(): Promise<boolean> {
|
||||
return await this.prompter.promptSingle<boolean>(<IQuestion>{
|
||||
type: QuestionTypes.confirm,
|
||||
message: localize('confirmReinstall', 'Are you sure you want to reinstall?'),
|
||||
default: true
|
||||
});
|
||||
}
|
||||
|
||||
public doManagePackages(): void {
|
||||
try {
|
||||
let terminal = this.apiWrapper.createTerminalWithOptions({ cwd: this.getPythonBinDir() });
|
||||
terminal.show(true);
|
||||
let shellType = this.apiWrapper.getConfiguration().get('terminal.integrated.shell.windows');
|
||||
terminal.sendText(this.getTextToSendToTerminal(shellType), true);
|
||||
} catch (error) {
|
||||
let message = utils.getErrorMessage(error);
|
||||
this.apiWrapper.showErrorMessage(message);
|
||||
}
|
||||
}
|
||||
|
||||
public async doConfigurePython(jupyterInstaller: JupyterServerInstallation): Promise<void> {
|
||||
try {
|
||||
let pythonDialog = new ConfigurePythonDialog(this.appContext, this.outputChannel, jupyterInstaller);
|
||||
await pythonDialog.showDialog();
|
||||
} catch (error) {
|
||||
let message = utils.getErrorMessage(error);
|
||||
this.apiWrapper.showErrorMessage(message);
|
||||
}
|
||||
}
|
||||
|
||||
public getTextToSendToTerminal(shellType: any): string {
|
||||
if (utils.getOSPlatform() === utils.Platform.Windows && typeof shellType === 'string') {
|
||||
if (shellType.endsWith('powershell.exe')) {
|
||||
return localizedConstants.msgManagePackagesPowershell;
|
||||
} else if (shellType.endsWith('cmd.exe')) {
|
||||
return localizedConstants.msgManagePackagesCmd;
|
||||
} else {
|
||||
return localizedConstants.msgManagePackagesBash;
|
||||
}
|
||||
} else {
|
||||
return localizedConstants.msgManagePackagesBash;
|
||||
}
|
||||
}
|
||||
|
||||
private getPythonBinDir(): string {
|
||||
return JupyterServerInstallation.getPythonBinPath(this.apiWrapper);
|
||||
}
|
||||
|
||||
public get jupyterInstallation() {
|
||||
return this._jupyterInstallation;
|
||||
}
|
||||
}
|
||||
172
extensions/notebook/src/jupyter/jupyterKernel.ts
Normal file
172
extensions/notebook/src/jupyter/jupyterKernel.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb } from 'sqlops';
|
||||
import { Kernel, KernelMessage } from '@jupyterlab/services';
|
||||
|
||||
function toShellMessage(msgImpl: KernelMessage.IShellMessage): nb.IShellMessage {
|
||||
return {
|
||||
channel: msgImpl.channel,
|
||||
type: msgImpl.channel,
|
||||
content: msgImpl.content,
|
||||
header: msgImpl.header,
|
||||
parent_header: msgImpl.parent_header,
|
||||
metadata: msgImpl.metadata
|
||||
};
|
||||
}
|
||||
|
||||
function toStdInMessage(msgImpl: KernelMessage.IStdinMessage): nb.IStdinMessage {
|
||||
return {
|
||||
channel: msgImpl.channel,
|
||||
type: msgImpl.channel,
|
||||
content: msgImpl.content,
|
||||
header: msgImpl.header,
|
||||
parent_header: msgImpl.parent_header,
|
||||
metadata: msgImpl.metadata
|
||||
};
|
||||
}
|
||||
|
||||
function toIOPubMessage(msgImpl: KernelMessage.IIOPubMessage): nb.IIOPubMessage {
|
||||
return {
|
||||
channel: msgImpl.channel,
|
||||
type: msgImpl.channel,
|
||||
content: msgImpl.content,
|
||||
header: msgImpl.header,
|
||||
parent_header: msgImpl.parent_header,
|
||||
metadata: msgImpl.metadata
|
||||
};
|
||||
}
|
||||
|
||||
function toIInputReply(content: nb.IInputReply): KernelMessage.IInputReply {
|
||||
return {
|
||||
value: content.value
|
||||
};
|
||||
}
|
||||
export class JupyterKernel implements nb.IKernel {
|
||||
constructor(private kernelImpl: Kernel.IKernelConnection) {
|
||||
}
|
||||
|
||||
public get id(): string {
|
||||
return this.kernelImpl.id;
|
||||
}
|
||||
|
||||
public get name(): string {
|
||||
return this.kernelImpl.name;
|
||||
}
|
||||
|
||||
public get supportsIntellisense(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
public get isReady(): boolean {
|
||||
return this.kernelImpl.isReady;
|
||||
}
|
||||
|
||||
public get ready(): Promise<void> {
|
||||
return this.kernelImpl.ready;
|
||||
}
|
||||
|
||||
public get info(): nb.IInfoReply {
|
||||
return this.kernelImpl.info as nb.IInfoReply;
|
||||
}
|
||||
|
||||
public async getSpec(): Promise<nb.IKernelSpec> {
|
||||
let specImpl = await this.kernelImpl.getSpec();
|
||||
return {
|
||||
name: specImpl.name,
|
||||
display_name: specImpl.display_name
|
||||
};
|
||||
}
|
||||
|
||||
requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture {
|
||||
let futureImpl = this.kernelImpl.requestExecute(content as KernelMessage.IExecuteRequest, disposeOnDone);
|
||||
return new JupyterFuture(futureImpl);
|
||||
}
|
||||
|
||||
requestComplete(content: nb.ICompleteRequest): Promise<nb.ICompleteReplyMsg> {
|
||||
return this.kernelImpl.requestComplete({
|
||||
code: content.code,
|
||||
cursor_pos: content.cursor_pos
|
||||
}).then((completeMsg) => {
|
||||
// Complete msg matches shell message definition, but with clearer content body
|
||||
let msg: nb.ICompleteReplyMsg = toShellMessage(completeMsg);
|
||||
return msg;
|
||||
});
|
||||
}
|
||||
|
||||
interrupt(): Promise<void> {
|
||||
return this.kernelImpl.interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
export class JupyterFuture implements nb.IFuture {
|
||||
|
||||
private _inProgress: boolean;
|
||||
|
||||
constructor(private futureImpl: Kernel.IFuture) {
|
||||
this._inProgress = true;
|
||||
}
|
||||
|
||||
public get msg(): nb.IShellMessage {
|
||||
let msgImpl = this.futureImpl.msg;
|
||||
return toShellMessage(msgImpl);
|
||||
}
|
||||
|
||||
public get done(): Promise<nb.IShellMessage> {
|
||||
// Convert on success, leave to throw original error on fail
|
||||
return this.futureImpl.done.then((msgImpl) => {
|
||||
this._inProgress = false;
|
||||
return toShellMessage(msgImpl);
|
||||
});
|
||||
}
|
||||
|
||||
public get inProgress(): boolean {
|
||||
return this._inProgress;
|
||||
}
|
||||
|
||||
public set inProgress(inProg: boolean) {
|
||||
this._inProgress = inProg;
|
||||
}
|
||||
|
||||
setReplyHandler(handler: nb.MessageHandler<nb.IShellMessage>): void {
|
||||
this.futureImpl.onReply = (msg) => {
|
||||
let shellMsg = toShellMessage(msg);
|
||||
return handler.handle(shellMsg);
|
||||
};
|
||||
}
|
||||
|
||||
setStdInHandler(handler: nb.MessageHandler<nb.IStdinMessage>): void {
|
||||
this.futureImpl.onStdin = (msg) => {
|
||||
let shellMsg = toStdInMessage(msg);
|
||||
return handler.handle(shellMsg);
|
||||
};
|
||||
}
|
||||
|
||||
setIOPubHandler(handler: nb.MessageHandler<nb.IIOPubMessage>): void {
|
||||
this.futureImpl.onIOPub = (msg) => {
|
||||
let shellMsg = toIOPubMessage(msg);
|
||||
return handler.handle(shellMsg);
|
||||
};
|
||||
}
|
||||
|
||||
registerMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | PromiseLike<boolean>): void {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
|
||||
removeMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | PromiseLike<boolean>): void {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
|
||||
sendInputReply(content: nb.IInputReply): void {
|
||||
this.futureImpl.sendInputReply(toIInputReply(content));
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.futureImpl.dispose();
|
||||
}
|
||||
}
|
||||
56
extensions/notebook/src/jupyter/jupyterNotebookManager.ts
Normal file
56
extensions/notebook/src/jupyter/jupyterNotebookManager.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb } from 'sqlops';
|
||||
import * as vscode from 'vscode';
|
||||
import { ServerConnection, SessionManager } from '@jupyterlab/services';
|
||||
|
||||
import { JupyterSessionManager } from './jupyterSessionManager';
|
||||
import { ApiWrapper } from '../common/apiWrapper';
|
||||
import { LocalJupyterServerManager } from './jupyterServerManager';
|
||||
|
||||
export class JupyterNotebookManager implements nb.NotebookManager, vscode.Disposable {
|
||||
protected _serverSettings: ServerConnection.ISettings;
|
||||
private _sessionManager: JupyterSessionManager;
|
||||
|
||||
constructor(private _serverManager: LocalJupyterServerManager, sessionManager?: JupyterSessionManager, private apiWrapper: ApiWrapper = new ApiWrapper()) {
|
||||
this._sessionManager = sessionManager || new JupyterSessionManager();
|
||||
this._serverManager.onServerStarted(() => {
|
||||
this.setServerSettings(this._serverManager.serverSettings);
|
||||
});
|
||||
|
||||
}
|
||||
public get contentManager(): nb.ContentManager {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
public get sessionManager(): nb.SessionManager {
|
||||
return this._sessionManager;
|
||||
}
|
||||
|
||||
public get serverManager(): nb.ServerManager {
|
||||
return this._serverManager;
|
||||
}
|
||||
|
||||
public get serverSettings(): ServerConnection.ISettings {
|
||||
return this._serverSettings;
|
||||
}
|
||||
|
||||
public setServerSettings(settings: Partial<ServerConnection.ISettings>): void {
|
||||
this._serverSettings = ServerConnection.makeSettings(settings);
|
||||
this._sessionManager.setJupyterSessionManager(new SessionManager({ serverSettings: this._serverSettings }));
|
||||
}
|
||||
|
||||
dispose() {
|
||||
if (this._sessionManager) {
|
||||
this._sessionManager.shutdownAll().then(() => this._sessionManager.dispose());
|
||||
}
|
||||
if (this._serverManager) {
|
||||
this._serverManager.stopServer().then(success => undefined, error => this.apiWrapper.showErrorMessage(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
82
extensions/notebook/src/jupyter/jupyterNotebookProvider.ts
Normal file
82
extensions/notebook/src/jupyter/jupyterNotebookProvider.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb } from 'sqlops';
|
||||
import * as vscode from 'vscode';
|
||||
import * as nls from 'vscode-nls';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import * as constants from '../common/constants';
|
||||
import { JupyterNotebookManager } from './jupyterNotebookManager';
|
||||
import { LocalJupyterServerManager } from './jupyterServerManager';
|
||||
|
||||
export type ServerManagerFactory = (documentUri: vscode.Uri) => LocalJupyterServerManager;
|
||||
|
||||
export class JupyterNotebookProvider implements nb.NotebookProvider {
|
||||
readonly providerId: string = constants.jupyterNotebookProviderId;
|
||||
private managerTracker = new Map<string, JupyterNotebookManager>();
|
||||
|
||||
constructor(private createServerManager: ServerManagerFactory) {
|
||||
}
|
||||
|
||||
public getNotebookManager(notebookUri: vscode.Uri): Thenable<nb.NotebookManager> {
|
||||
if (!notebookUri) {
|
||||
return Promise.reject(localize('errNotebookUriMissing', 'A notebook path is required'));
|
||||
}
|
||||
return this.doGetNotebookManager(notebookUri);
|
||||
}
|
||||
|
||||
private doGetNotebookManager(notebookUri: vscode.Uri): Promise<nb.NotebookManager> {
|
||||
let uriString = notebookUri.toString();
|
||||
let manager = this.managerTracker.get(uriString);
|
||||
if (!manager) {
|
||||
let serverManager = this.createServerManager(notebookUri);
|
||||
manager = new JupyterNotebookManager(serverManager);
|
||||
this.managerTracker.set(uriString, manager);
|
||||
}
|
||||
return Promise.resolve(manager);
|
||||
}
|
||||
|
||||
handleNotebookClosed(notebookUri: vscode.Uri): void {
|
||||
if (!notebookUri) {
|
||||
// As this is a notification method, will skip throwing an error here
|
||||
return;
|
||||
}
|
||||
let uriString = notebookUri.toString();
|
||||
let manager = this.managerTracker.get(uriString);
|
||||
if (manager) {
|
||||
this.managerTracker.delete(uriString);
|
||||
manager.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public get standardKernels(): nb.IStandardKernel[] {
|
||||
return [
|
||||
{
|
||||
"name": "Python 3",
|
||||
"connectionProviderIds": []
|
||||
},
|
||||
{
|
||||
"name": "PySpark",
|
||||
"connectionProviderIds": ["HADOOP_KNOX"]
|
||||
},
|
||||
{
|
||||
"name": "PySpark3",
|
||||
"connectionProviderIds": ["HADOOP_KNOX"]
|
||||
},
|
||||
{
|
||||
"name": "Spark | R",
|
||||
"connectionProviderIds": ["HADOOP_KNOX"]
|
||||
},
|
||||
{
|
||||
"name": "Spark | Scala",
|
||||
"connectionProviderIds": ["HADOOP_KNOX"]
|
||||
}
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
338
extensions/notebook/src/jupyter/jupyterServerInstallation.ts
Normal file
338
extensions/notebook/src/jupyter/jupyterServerInstallation.ts
Normal file
@@ -0,0 +1,338 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import * as nls from 'vscode-nls';
|
||||
import * as sqlops from 'sqlops';
|
||||
import { ExecOptions } from 'child_process';
|
||||
import * as decompress from 'decompress';
|
||||
import * as request from 'request';
|
||||
|
||||
import { ApiWrapper } from '../common/apiWrapper';
|
||||
import * as constants from '../common/constants';
|
||||
import * as utils from '../common/utils';
|
||||
import { OutputChannel, ConfigurationTarget, Event, EventEmitter, window } from 'vscode';
|
||||
|
||||
const localize = nls.loadMessageBundle();
|
||||
const msgPythonInstallationProgress = localize('msgPythonInstallationProgress', 'Python installation is in progress');
|
||||
const msgPythonInstallationComplete = localize('msgPythonInstallationComplete', 'Python installation is complete');
|
||||
const msgPythonDownloadError = localize('msgPythonDownloadError', 'Error while downloading python setup');
|
||||
const msgPythonDownloadPending = localize('msgPythonDownloadPending', 'Downloading python package');
|
||||
const msgPythonUnpackPending = localize('msgPythonUnpackPending', 'Unpacking python package');
|
||||
const msgPythonDirectoryError = localize('msgPythonDirectoryError', 'Error while creating python installation directory');
|
||||
const msgPythonUnpackError = localize('msgPythonUnpackError', 'Error while unpacking python bundle');
|
||||
const msgTaskName = localize('msgTaskName', 'Installing Notebook dependencies');
|
||||
const msgInstallPkgStart = localize('msgInstallPkgStart', 'Installing Notebook dependencies, see Tasks view for more information');
|
||||
const msgInstallPkgFinish = localize('msgInstallPkgFinish', 'Notebook dependencies installation is complete');
|
||||
function msgDependenciesInstallationFailed(errorMessage: string): string { return localize('msgDependenciesInstallationFailed', 'Installing Notebook dependencies failed with error: {0}', errorMessage); }
|
||||
function msgDownloadPython(platform: string, pythonDownloadUrl: string): string { return localize('msgDownloadPython', 'Downloading local python for platform: {0} to {1}', platform, pythonDownloadUrl); }
|
||||
|
||||
export default class JupyterServerInstallation {
|
||||
/**
|
||||
* Path to the folder where all configuration sets will be stored. Should always be:
|
||||
* %extension_path%/jupyter_config
|
||||
*/
|
||||
public apiWrapper: ApiWrapper;
|
||||
public extensionPath: string;
|
||||
public pythonBinPath: string;
|
||||
public outputChannel: OutputChannel;
|
||||
public configRoot: string;
|
||||
public pythonEnvVarPath: string;
|
||||
public execOptions: ExecOptions;
|
||||
|
||||
private _pythonInstallationPath: string;
|
||||
private _pythonExecutable: string;
|
||||
|
||||
// Allows dependencies to be installed even if an existing installation is already present
|
||||
private _forceInstall: boolean;
|
||||
|
||||
private static readonly DefaultPythonLocation = path.join(utils.getUserHome(), 'azuredatastudio-python');
|
||||
|
||||
private _installCompleteEmitter = new EventEmitter<string>();
|
||||
|
||||
constructor(extensionPath: string, outputChannel: OutputChannel, apiWrapper: ApiWrapper, pythonInstallationPath?: string, forceInstall?: boolean) {
|
||||
this.extensionPath = extensionPath;
|
||||
this.outputChannel = outputChannel;
|
||||
this.apiWrapper = apiWrapper;
|
||||
this._pythonInstallationPath = pythonInstallationPath || JupyterServerInstallation.getPythonInstallPath(this.apiWrapper);
|
||||
this.configRoot = path.join(this.extensionPath, constants.jupyterConfigRootFolder);
|
||||
this._forceInstall = !!forceInstall;
|
||||
|
||||
this.configurePackagePaths();
|
||||
}
|
||||
|
||||
public get onInstallComplete(): Event<string> {
|
||||
return this._installCompleteEmitter.event;
|
||||
}
|
||||
|
||||
public static async getInstallation(
|
||||
extensionPath: string,
|
||||
outputChannel: OutputChannel,
|
||||
apiWrapper: ApiWrapper,
|
||||
pythonInstallationPath?: string,
|
||||
forceInstall?: boolean): Promise<JupyterServerInstallation> {
|
||||
|
||||
let installation = new JupyterServerInstallation(extensionPath, outputChannel, apiWrapper, pythonInstallationPath, forceInstall);
|
||||
await installation.startInstallProcess();
|
||||
|
||||
return installation;
|
||||
}
|
||||
|
||||
private async installDependencies(backgroundOperation: sqlops.BackgroundOperation): Promise<void> {
|
||||
if (!fs.existsSync(this._pythonExecutable) || this._forceInstall) {
|
||||
window.showInformationMessage(msgInstallPkgStart);
|
||||
this.outputChannel.show(true);
|
||||
this.outputChannel.appendLine(msgPythonInstallationProgress);
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonInstallationProgress);
|
||||
await this.installPythonPackage(backgroundOperation);
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonInstallationComplete);
|
||||
this.outputChannel.appendLine(msgPythonInstallationComplete);
|
||||
|
||||
// Install jupyter on Windows because local python is not bundled with jupyter unlike linux and MacOS.
|
||||
await this.installJupyterProsePackage();
|
||||
await this.installSparkMagic();
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.Succeeded, msgInstallPkgFinish);
|
||||
window.showInformationMessage(msgInstallPkgFinish);
|
||||
}
|
||||
}
|
||||
|
||||
private installPythonPackage(backgroundOperation: sqlops.BackgroundOperation): Promise<void> {
|
||||
let bundleVersion = constants.pythonBundleVersion;
|
||||
let pythonVersion = constants.pythonVersion;
|
||||
let packageName = 'python-#pythonversion-#platform-#bundleversion.#extension';
|
||||
let platformId = utils.getOSPlatformId();
|
||||
|
||||
packageName = packageName.replace('#platform', platformId)
|
||||
.replace('#pythonversion', pythonVersion)
|
||||
.replace('#bundleversion', bundleVersion)
|
||||
.replace('#extension', process.platform === constants.winPlatform ? 'zip' : 'tar.gz');
|
||||
|
||||
let pythonDownloadUrl = undefined;
|
||||
switch (utils.getOSPlatform()) {
|
||||
case utils.Platform.Windows:
|
||||
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065977';
|
||||
break;
|
||||
case utils.Platform.Mac:
|
||||
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065976';
|
||||
break;
|
||||
default:
|
||||
// Default to linux
|
||||
pythonDownloadUrl = 'https://go.microsoft.com/fwlink/?linkid=2065975';
|
||||
break;
|
||||
}
|
||||
|
||||
let pythonPackagePathLocal = this._pythonInstallationPath + '/' + packageName;
|
||||
let self = undefined;
|
||||
return new Promise((resolve, reject) => {
|
||||
self = this;
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgDownloadPython(platformId, pythonDownloadUrl));
|
||||
fs.mkdirs(this._pythonInstallationPath, (err) => {
|
||||
if (err) {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDirectoryError);
|
||||
reject(err);
|
||||
}
|
||||
|
||||
let totalMegaBytes: number = undefined;
|
||||
let receivedBytes = 0;
|
||||
let printThreshold = 0.1;
|
||||
request.get(pythonDownloadUrl, { timeout: 20000 })
|
||||
.on('error', (downloadError) => {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
|
||||
reject(downloadError);
|
||||
})
|
||||
.on('response', (response) => {
|
||||
if (response.statusCode !== 200) {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
|
||||
reject(response.statusMessage);
|
||||
}
|
||||
|
||||
let totalBytes = parseInt(response.headers['content-length']);
|
||||
totalMegaBytes = totalBytes / (1024 * 1024);
|
||||
this.outputChannel.appendLine(`${msgPythonDownloadPending} (0 / ${totalMegaBytes.toFixed(2)} MB)`);
|
||||
})
|
||||
.on('data', (data) => {
|
||||
receivedBytes += data.length;
|
||||
if (totalMegaBytes) {
|
||||
let receivedMegaBytes = receivedBytes / (1024 * 1024);
|
||||
let percentage = receivedMegaBytes / totalMegaBytes;
|
||||
if (percentage >= printThreshold) {
|
||||
this.outputChannel.appendLine(`${msgPythonDownloadPending} (${receivedMegaBytes.toFixed(2)} / ${totalMegaBytes.toFixed(2)} MB)`);
|
||||
printThreshold += 0.1;
|
||||
}
|
||||
}
|
||||
})
|
||||
.pipe(fs.createWriteStream(pythonPackagePathLocal))
|
||||
.on('close', () => {
|
||||
//unpack python zip/tar file
|
||||
this.outputChannel.appendLine(msgPythonUnpackPending);
|
||||
let pythonSourcePath = path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
|
||||
if (fs.existsSync(pythonSourcePath)) {
|
||||
try {
|
||||
fs.removeSync(pythonSourcePath);
|
||||
} catch (err) {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
|
||||
reject(err);
|
||||
}
|
||||
}
|
||||
decompress(pythonPackagePathLocal, self._pythonInstallationPath).then(files => {
|
||||
//Delete zip/tar file
|
||||
fs.unlink(pythonPackagePathLocal, (err) => {
|
||||
if (err) {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
resolve();
|
||||
}).catch(err => {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonUnpackError);
|
||||
reject(err);
|
||||
});
|
||||
})
|
||||
.on('error', (downloadError) => {
|
||||
backgroundOperation.updateStatus(sqlops.TaskStatus.InProgress, msgPythonDownloadError);
|
||||
reject(downloadError);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private configurePackagePaths(): void {
|
||||
//Python source path up to bundle version
|
||||
let pythonSourcePath = path.join(this._pythonInstallationPath, constants.pythonBundleVersion);
|
||||
|
||||
// Update python paths and properties to reference user's local python.
|
||||
let pythonBinPathSuffix = process.platform === constants.winPlatform ? '' : 'bin';
|
||||
|
||||
this._pythonExecutable = path.join(pythonSourcePath, process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
|
||||
this.pythonBinPath = path.join(pythonSourcePath, pythonBinPathSuffix);
|
||||
|
||||
// Store paths to python libraries required to run jupyter.
|
||||
this.pythonEnvVarPath = process.env.Path;
|
||||
|
||||
let delimiter = path.delimiter;
|
||||
if (process.platform === constants.winPlatform) {
|
||||
let pythonScriptsPath = path.join(pythonSourcePath, 'Scripts');
|
||||
this.pythonEnvVarPath = pythonScriptsPath + delimiter + this.pythonEnvVarPath;
|
||||
}
|
||||
this.pythonEnvVarPath = this.pythonBinPath + delimiter + this.pythonEnvVarPath;
|
||||
|
||||
// Store the executable options to run child processes with env var without interfering parent env var.
|
||||
let env = Object.assign({}, process.env);
|
||||
env['PATH'] = this.pythonEnvVarPath;
|
||||
this.execOptions = {
|
||||
env: env
|
||||
};
|
||||
}
|
||||
|
||||
public async startInstallProcess(pythonInstallationPath?: string): Promise<void> {
|
||||
if (pythonInstallationPath) {
|
||||
this._pythonInstallationPath = pythonInstallationPath;
|
||||
this.configurePackagePaths();
|
||||
}
|
||||
let updateConfig = () => {
|
||||
let notebookConfig = this.apiWrapper.getConfiguration(constants.notebookConfigKey);
|
||||
notebookConfig.update(constants.pythonPathConfigKey, this._pythonInstallationPath, ConfigurationTarget.Global);
|
||||
};
|
||||
if (!fs.existsSync(this._pythonExecutable) || this._forceInstall) {
|
||||
this.apiWrapper.startBackgroundOperation({
|
||||
displayName: msgTaskName,
|
||||
description: msgTaskName,
|
||||
isCancelable: false,
|
||||
operation: op => {
|
||||
this.installDependencies(op)
|
||||
.then(() => {
|
||||
this._installCompleteEmitter.fire();
|
||||
updateConfig();
|
||||
})
|
||||
.catch(err => {
|
||||
let errorMsg = msgDependenciesInstallationFailed(err);
|
||||
op.updateStatus(sqlops.TaskStatus.Failed, errorMsg);
|
||||
this.apiWrapper.showErrorMessage(errorMsg);
|
||||
this._installCompleteEmitter.fire(errorMsg);
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Python executable already exists, but the path setting wasn't defined,
|
||||
// so update it here
|
||||
this._installCompleteEmitter.fire();
|
||||
updateConfig();
|
||||
}
|
||||
}
|
||||
|
||||
private async installJupyterProsePackage(): Promise<void> {
|
||||
if (process.platform === constants.winPlatform) {
|
||||
let installJupyterCommand = `${this._pythonExecutable} -m pip install pandas==0.22.0 jupyter prose-codeaccelerator==1.3.0 --extra-index-url https://prose-python-packages.azurewebsites.net --no-warn-script-location`;
|
||||
this.outputChannel.show(true);
|
||||
this.outputChannel.appendLine(localize('msgInstallStart', 'Installing required packages to run Notebooks...'));
|
||||
await utils.executeStreamedCommand(installJupyterCommand, this.outputChannel);
|
||||
this.outputChannel.appendLine(localize('msgJupyterInstallDone', '... Jupyter installation complete.'));
|
||||
} else {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
private async installSparkMagic(): Promise<void> {
|
||||
if (process.platform === constants.winPlatform) {
|
||||
let sparkMagicPath = path.join(this.extensionPath, 'wheels/sparkmagic-#sparkMagicVersion-py3-none-any.whl'.replace('#sparkMagicVersion', constants.sparkMagicVersion));
|
||||
let installSparkMagic = `${this._pythonExecutable} -m pip install ${sparkMagicPath} --no-warn-script-location`;
|
||||
this.outputChannel.show(true);
|
||||
this.outputChannel.appendLine(localize('msgInstallingSpark', 'Installing SparkMagic...'));
|
||||
await utils.executeStreamedCommand(installSparkMagic, this.outputChannel);
|
||||
} else {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
public get pythonExecutable(): string {
|
||||
return this._pythonExecutable;
|
||||
}
|
||||
|
||||
public static isPythonInstalled(apiWrapper: ApiWrapper): boolean {
|
||||
// Don't use _pythonExecutable here, since it could be populated with a default value
|
||||
let pathSetting = JupyterServerInstallation.getPythonPathSetting(apiWrapper);
|
||||
if (!pathSetting) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let pythonExe = path.join(
|
||||
pathSetting,
|
||||
constants.pythonBundleVersion,
|
||||
process.platform === constants.winPlatform ? 'python.exe' : 'bin/python3');
|
||||
return fs.existsSync(pythonExe);
|
||||
}
|
||||
|
||||
public static getPythonInstallPath(apiWrapper: ApiWrapper): string {
|
||||
let userPath = JupyterServerInstallation.getPythonPathSetting(apiWrapper);
|
||||
return userPath ? userPath : JupyterServerInstallation.DefaultPythonLocation;
|
||||
}
|
||||
|
||||
private static getPythonPathSetting(apiWrapper: ApiWrapper): string {
|
||||
let path = undefined;
|
||||
if (apiWrapper) {
|
||||
let notebookConfig = apiWrapper.getConfiguration(constants.notebookConfigKey);
|
||||
if (notebookConfig) {
|
||||
let configPythonPath = notebookConfig[constants.pythonPathConfigKey];
|
||||
if (configPythonPath && fs.existsSync(configPythonPath)) {
|
||||
path = configPythonPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
public static getPythonBinPath(apiWrapper: ApiWrapper): string {
|
||||
let pythonBinPathSuffix = process.platform === constants.winPlatform ? '' : 'bin';
|
||||
|
||||
return path.join(
|
||||
JupyterServerInstallation.getPythonInstallPath(apiWrapper),
|
||||
constants.pythonBundleVersion,
|
||||
pythonBinPathSuffix);
|
||||
}
|
||||
}
|
||||
147
extensions/notebook/src/jupyter/jupyterServerManager.ts
Normal file
147
extensions/notebook/src/jupyter/jupyterServerManager.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb } from 'sqlops';
|
||||
import * as vscode from 'vscode';
|
||||
import * as path from 'path';
|
||||
import { ServerConnection } from '@jupyterlab/services';
|
||||
import * as nls from 'vscode-nls';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import { ApiWrapper } from '../common/apiWrapper';
|
||||
import JupyterServerInstallation from './jupyterServerInstallation';
|
||||
import * as utils from '../common/utils';
|
||||
import { IServerInstance } from './common';
|
||||
import { PerNotebookServerInstance, IInstanceOptions } from './serverInstance';
|
||||
|
||||
export interface IServerManagerOptions {
|
||||
documentPath: string;
|
||||
jupyterInstallation: Promise<JupyterServerInstallation>;
|
||||
extensionContext: vscode.ExtensionContext;
|
||||
apiWrapper?: ApiWrapper;
|
||||
factory?: ServerInstanceFactory;
|
||||
}
|
||||
export class LocalJupyterServerManager implements nb.ServerManager, vscode.Disposable {
|
||||
private _serverSettings: Partial<ServerConnection.ISettings>;
|
||||
private _onServerStarted = new vscode.EventEmitter<void>();
|
||||
private _instanceOptions: IInstanceOptions;
|
||||
private apiWrapper: ApiWrapper;
|
||||
private jupyterServer: IServerInstance;
|
||||
factory: ServerInstanceFactory;
|
||||
constructor(private options: IServerManagerOptions) {
|
||||
this.apiWrapper = options.apiWrapper || new ApiWrapper();
|
||||
this.factory = options.factory || new ServerInstanceFactory();
|
||||
}
|
||||
|
||||
public get serverSettings(): Partial<ServerConnection.ISettings> {
|
||||
return this._serverSettings;
|
||||
}
|
||||
|
||||
public get isStarted(): boolean {
|
||||
return !!this.jupyterServer;
|
||||
}
|
||||
|
||||
public get instanceOptions(): IInstanceOptions {
|
||||
return this._instanceOptions;
|
||||
}
|
||||
|
||||
public get onServerStarted(): vscode.Event<void> {
|
||||
return this._onServerStarted.event;
|
||||
}
|
||||
|
||||
public async startServer(): Promise<void> {
|
||||
try {
|
||||
this.jupyterServer = await this.doStartServer();
|
||||
this.options.extensionContext.subscriptions.push(this);
|
||||
let partialSettings = LocalJupyterServerManager.getLocalConnectionSettings(this.jupyterServer.uri);
|
||||
this._serverSettings = partialSettings;
|
||||
this._onServerStarted.fire();
|
||||
|
||||
} catch (error) {
|
||||
this.apiWrapper.showErrorMessage(localize('startServerFailed', 'Starting local Notebook server failed with error {0}', utils.getErrorMessage(error)));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this.stopServer().catch(err => {
|
||||
let msg = utils.getErrorMessage(err);
|
||||
this.apiWrapper.showErrorMessage(localize('shutdownError', 'Shutdown of Notebook server failed: {0}', msg));
|
||||
});
|
||||
}
|
||||
|
||||
public async stopServer(): Promise<void> {
|
||||
if (this.jupyterServer) {
|
||||
await this.jupyterServer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
public static getLocalConnectionSettings(uri: vscode.Uri): Partial<ServerConnection.ISettings> {
|
||||
return {
|
||||
baseUrl: `${uri.scheme}://${uri.authority}`,
|
||||
token: LocalJupyterServerManager.getToken(uri.query)
|
||||
};
|
||||
}
|
||||
|
||||
private static getToken(query: string): string {
|
||||
if (query) {
|
||||
let parts = query.split('=');
|
||||
if (parts && parts.length >= 2) {
|
||||
return parts[1];
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
private get documentPath(): string {
|
||||
return this.options.documentPath;
|
||||
}
|
||||
|
||||
private async doStartServer(): Promise<IServerInstance> { // We can't find or create servers until the installation is complete
|
||||
let installation = await this.options.jupyterInstallation;
|
||||
|
||||
// Calculate the path to use as the notebook-dir for Jupyter based on the path of the uri of the
|
||||
// notebook to open. This will be the workspace folder if the notebook uri is inside a workspace
|
||||
// folder. Otherwise, it will be the folder that the notebook is inside. Ultimately, this means
|
||||
// a new notebook server will be started for each folder a notebook is opened from.
|
||||
//
|
||||
// eg, opening:
|
||||
// /path1/nb1.ipynb
|
||||
// /path2/nb2.ipynb
|
||||
// /path2/nb3.ipynb
|
||||
// ... will result in 2 notebook servers being started, one for /path1/ and one for /path2/
|
||||
let notebookDir = this.apiWrapper.getWorkspacePathFromUri(vscode.Uri.file(this.documentPath));
|
||||
notebookDir = notebookDir || path.dirname(this.documentPath);
|
||||
|
||||
// TODO handle notification of start/stop status
|
||||
// notebookContext.updateLoadingMessage(localizedConstants.msgJupyterStarting);
|
||||
|
||||
// TODO refactor server instance so it doesn't need the kernel. Likely need to reimplement this
|
||||
// for notebook version
|
||||
let serverInstanceOptions: IInstanceOptions = {
|
||||
documentPath: this.documentPath,
|
||||
notebookDirectory: notebookDir,
|
||||
install: installation
|
||||
};
|
||||
|
||||
this._instanceOptions = serverInstanceOptions;
|
||||
|
||||
let server = this.factory.createInstance(serverInstanceOptions);
|
||||
await server.configure();
|
||||
await server.start();
|
||||
|
||||
return server;
|
||||
}
|
||||
}
|
||||
|
||||
export class ServerInstanceFactory {
|
||||
|
||||
createInstance(options: IInstanceOptions): IServerInstance {
|
||||
return new PerNotebookServerInstance(options);
|
||||
}
|
||||
}
|
||||
|
||||
338
extensions/notebook/src/jupyter/jupyterSessionManager.ts
Normal file
338
extensions/notebook/src/jupyter/jupyterSessionManager.ts
Normal file
@@ -0,0 +1,338 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb, ServerInfo, connection, IConnectionProfile } from 'sqlops';
|
||||
import { Session, Kernel } from '@jupyterlab/services';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as nls from 'vscode-nls';
|
||||
import { Uri } from 'vscode';
|
||||
import * as path from 'path';
|
||||
import * as utils from '../common/utils';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import { JupyterKernel } from './jupyterKernel';
|
||||
import { Deferred } from '../common/promise';
|
||||
|
||||
const configBase = {
|
||||
'kernel_python_credentials': {
|
||||
'url': ''
|
||||
},
|
||||
'kernel_scala_credentials': {
|
||||
'url': ''
|
||||
},
|
||||
'kernel_r_credentials': {
|
||||
'url': ''
|
||||
},
|
||||
|
||||
'ignore_ssl_errors': true,
|
||||
|
||||
'logging_config': {
|
||||
'version': 1,
|
||||
'formatters': {
|
||||
'magicsFormatter': {
|
||||
'format': '%(asctime)s\t%(levelname)s\t%(message)s',
|
||||
'datefmt': ''
|
||||
}
|
||||
},
|
||||
'handlers': {
|
||||
'magicsHandler': {
|
||||
'class': 'hdijupyterutils.filehandler.MagicsFileHandler',
|
||||
'formatter': 'magicsFormatter',
|
||||
'home_path': ''
|
||||
}
|
||||
},
|
||||
'loggers': {
|
||||
'magicsLogger': {
|
||||
'handlers': ['magicsHandler'],
|
||||
'level': 'DEBUG',
|
||||
'propagate': 0
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const KNOX_ENDPOINT_SERVER = 'host';
|
||||
const KNOX_ENDPOINT_PORT = 'knoxport';
|
||||
const KNOX_ENDPOINT = 'knox';
|
||||
const SQL_PROVIDER = 'MSSQL';
|
||||
const USER = 'user';
|
||||
const DEFAULT_CLUSTER_USER_NAME = 'root';
|
||||
|
||||
export class JupyterSessionManager implements nb.SessionManager {
|
||||
private _ready: Deferred<void>;
|
||||
private _isReady: boolean;
|
||||
private _sessionManager: Session.IManager;
|
||||
private static _sessions: JupyterSession[] = [];
|
||||
|
||||
constructor() {
|
||||
this._isReady = false;
|
||||
this._ready = new Deferred<void>();
|
||||
}
|
||||
|
||||
public setJupyterSessionManager(sessionManager: Session.IManager): void {
|
||||
this._sessionManager = sessionManager;
|
||||
sessionManager.ready
|
||||
.then(() => {
|
||||
this._isReady = true;
|
||||
this._ready.resolve();
|
||||
}).catch((error) => {
|
||||
this._isReady = false;
|
||||
this._ready.reject(error);
|
||||
});
|
||||
}
|
||||
|
||||
public get isReady(): boolean {
|
||||
return this._isReady;
|
||||
}
|
||||
public get ready(): Promise<void> {
|
||||
return this._ready.promise;
|
||||
}
|
||||
|
||||
public get specs(): nb.IAllKernels | undefined {
|
||||
if (!this._isReady) {
|
||||
return undefined;
|
||||
}
|
||||
let specs = this._sessionManager.specs;
|
||||
if (!specs) {
|
||||
return undefined;
|
||||
}
|
||||
let kernels: nb.IKernelSpec[] = Object.keys(specs.kernelspecs).map(k => {
|
||||
let value = specs.kernelspecs[k];
|
||||
let kernel: nb.IKernelSpec = {
|
||||
name: k,
|
||||
display_name: value.display_name ? value.display_name : k
|
||||
};
|
||||
// TODO add more info to kernels
|
||||
return kernel;
|
||||
});
|
||||
let allKernels: nb.IAllKernels = {
|
||||
defaultKernel: specs.default,
|
||||
kernels: kernels
|
||||
};
|
||||
return allKernels;
|
||||
}
|
||||
|
||||
public async startNew(options: nb.ISessionOptions): Promise<nb.ISession> {
|
||||
if (!this._isReady) {
|
||||
// no-op
|
||||
return Promise.reject(new Error(localize('errorStartBeforeReady', 'Cannot start a session, the manager is not yet initialized')));
|
||||
}
|
||||
let sessionImpl = await this._sessionManager.startNew(options);
|
||||
let jupyterSession = new JupyterSession(sessionImpl);
|
||||
let index = JupyterSessionManager._sessions.findIndex(session => session.path === options.path);
|
||||
if (index > -1) {
|
||||
JupyterSessionManager._sessions.splice(index);
|
||||
}
|
||||
JupyterSessionManager._sessions.push(jupyterSession);
|
||||
return jupyterSession;
|
||||
}
|
||||
|
||||
public listRunning(): JupyterSession[] {
|
||||
return JupyterSessionManager._sessions;
|
||||
}
|
||||
|
||||
public shutdown(id: string): Promise<void> {
|
||||
if (!this._isReady) {
|
||||
// no-op
|
||||
return Promise.resolve();
|
||||
}
|
||||
let index = JupyterSessionManager._sessions.findIndex(session => session.id === id);
|
||||
if (index > -1) {
|
||||
JupyterSessionManager._sessions.splice(index);
|
||||
}
|
||||
if (this._sessionManager && !this._sessionManager.isDisposed) {
|
||||
return this._sessionManager.shutdown(id);
|
||||
}
|
||||
}
|
||||
|
||||
public shutdownAll(): Promise<void> {
|
||||
return this._sessionManager.shutdownAll();
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this._sessionManager.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
export class JupyterSession implements nb.ISession {
|
||||
private _kernel: nb.IKernel;
|
||||
|
||||
constructor(private sessionImpl: Session.ISession) {
|
||||
}
|
||||
|
||||
public get canChangeKernels(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
public get id(): string {
|
||||
return this.sessionImpl.id;
|
||||
}
|
||||
|
||||
public get path(): string {
|
||||
return this.sessionImpl.path;
|
||||
}
|
||||
|
||||
public get name(): string {
|
||||
return this.sessionImpl.name;
|
||||
}
|
||||
|
||||
public get type(): string {
|
||||
return this.sessionImpl.type;
|
||||
}
|
||||
|
||||
public get status(): nb.KernelStatus {
|
||||
return this.sessionImpl.status;
|
||||
}
|
||||
|
||||
public get kernel(): nb.IKernel {
|
||||
if (!this._kernel) {
|
||||
let kernelImpl = this.sessionImpl.kernel;
|
||||
if (kernelImpl) {
|
||||
this._kernel = new JupyterKernel(kernelImpl);
|
||||
}
|
||||
}
|
||||
return this._kernel;
|
||||
}
|
||||
|
||||
public async changeKernel(kernelInfo: nb.IKernelSpec): Promise<nb.IKernel> {
|
||||
// For now, Jupyter implementation handles disposal etc. so we can just
|
||||
// null out our kernel and let the changeKernel call handle this
|
||||
this._kernel = undefined;
|
||||
// For now, just using name. It's unclear how we'd know the ID
|
||||
let options: Partial<Kernel.IModel> = {
|
||||
name: kernelInfo.name
|
||||
};
|
||||
return this.sessionImpl.changeKernel(options).then((kernelImpl) => {
|
||||
this._kernel = new JupyterKernel(kernelImpl);
|
||||
return this._kernel;
|
||||
});
|
||||
}
|
||||
|
||||
public async configureKernel(): Promise<void> {
|
||||
let sparkmagicConfDir = path.join(utils.getUserHome(), '.sparkmagic');
|
||||
await utils.mkDir(sparkmagicConfDir);
|
||||
|
||||
// Default to localhost in config file.
|
||||
let creds: ICredentials = {
|
||||
'url': 'http://localhost:8088'
|
||||
};
|
||||
|
||||
let config: ISparkMagicConfig = Object.assign({}, configBase);
|
||||
this.updateConfig(config, creds, sparkmagicConfDir);
|
||||
|
||||
let configFilePath = path.join(sparkmagicConfDir, 'config.json');
|
||||
await fs.writeFile(configFilePath, JSON.stringify(config));
|
||||
}
|
||||
|
||||
public async configureConnection(connection: IConnectionProfile): Promise<void> {
|
||||
if (connection && connection.providerName && this.isSparkKernel(this.sessionImpl.kernel.name)) {
|
||||
// TODO may need to reenable a way to get the credential
|
||||
// await this._connection.getCredential();
|
||||
// %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options,
|
||||
// such as user/profile/host name/auth type
|
||||
|
||||
//Update server info with bigdata endpoint - Unified Connection
|
||||
if (connection.providerName === SQL_PROVIDER) {
|
||||
let clusterEndpoint: IEndpoint = await this.getClusterEndpoint(connection.id, KNOX_ENDPOINT);
|
||||
if (!clusterEndpoint) {
|
||||
let kernelDisplayName: string = await this.getKernelDisplayName();
|
||||
return Promise.reject(new Error(localize('connectionNotValid', 'Spark kernels require a connection to a SQL Server big data cluster master instance.')));
|
||||
}
|
||||
connection.options[KNOX_ENDPOINT_SERVER] = clusterEndpoint.ipAddress;
|
||||
connection.options[KNOX_ENDPOINT_PORT] = clusterEndpoint.port;
|
||||
connection.options[USER] = DEFAULT_CLUSTER_USER_NAME;
|
||||
}
|
||||
else {
|
||||
connection.options[KNOX_ENDPOINT_PORT] = this.getKnoxPortOrDefault(connection);
|
||||
}
|
||||
this.setHostAndPort(':', connection);
|
||||
this.setHostAndPort(',', connection);
|
||||
|
||||
let server = Uri.parse(utils.getLivyUrl(connection.options[KNOX_ENDPOINT_SERVER], connection.options[KNOX_ENDPOINT_PORT])).toString();
|
||||
let doNotCallChangeEndpointParams =
|
||||
`%_do_not_call_change_endpoint --username=${connection.options[USER]} --password=${connection.options['password']} --server=${server} --auth=Basic_Access`;
|
||||
let future = this.sessionImpl.kernel.requestExecute({
|
||||
code: doNotCallChangeEndpointParams
|
||||
}, true);
|
||||
await future.done;
|
||||
}
|
||||
}
|
||||
|
||||
private async getKernelDisplayName(): Promise<string> {
|
||||
let spec = await this.kernel.getSpec();
|
||||
return spec.display_name;
|
||||
}
|
||||
|
||||
private isSparkKernel(kernelName: string): boolean {
|
||||
return kernelName && kernelName.toLowerCase().indexOf('spark') > -1;
|
||||
}
|
||||
|
||||
private setHostAndPort(delimeter: string, connection: IConnectionProfile): void {
|
||||
let originalHost = connection.options[KNOX_ENDPOINT_SERVER];
|
||||
if (!originalHost) {
|
||||
return;
|
||||
}
|
||||
let index = originalHost.indexOf(delimeter);
|
||||
if (index > -1) {
|
||||
connection.options[KNOX_ENDPOINT_SERVER] = originalHost.slice(0, index);
|
||||
connection.options[KNOX_ENDPOINT_PORT] = originalHost.slice(index + 1);
|
||||
}
|
||||
}
|
||||
|
||||
private updateConfig(config: ISparkMagicConfig, creds: ICredentials, homePath: string): void {
|
||||
config.kernel_python_credentials = creds;
|
||||
config.kernel_scala_credentials = creds;
|
||||
config.kernel_r_credentials = creds;
|
||||
config.logging_config.handlers.magicsHandler.home_path = homePath;
|
||||
}
|
||||
|
||||
private getKnoxPortOrDefault(connectionProfile: IConnectionProfile): string {
|
||||
let port = connectionProfile.options[KNOX_ENDPOINT_PORT];
|
||||
if (!port) {
|
||||
port = '30443';
|
||||
}
|
||||
return port;
|
||||
}
|
||||
|
||||
private async getClusterEndpoint(profileId: string, serviceName: string): Promise<IEndpoint> {
|
||||
let serverInfo: ServerInfo = await connection.getServerInfo(profileId);
|
||||
if (!serverInfo || !serverInfo.options) {
|
||||
return undefined;
|
||||
}
|
||||
let endpoints: IEndpoint[] = serverInfo.options['clusterEndpoints'];
|
||||
if (!endpoints || endpoints.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return endpoints.find(ep => ep.serviceName.toLowerCase() === serviceName.toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
interface ICredentials {
|
||||
'url': string;
|
||||
}
|
||||
|
||||
interface IEndpoint {
|
||||
serviceName: string;
|
||||
ipAddress: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
interface ISparkMagicConfig {
|
||||
kernel_python_credentials: ICredentials;
|
||||
kernel_scala_credentials: ICredentials;
|
||||
kernel_r_credentials: ICredentials;
|
||||
ignore_ssl_errors?: boolean;
|
||||
logging_config: {
|
||||
handlers: {
|
||||
magicsHandler: {
|
||||
home_path: string;
|
||||
class?: string;
|
||||
formatter?: string
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
76
extensions/notebook/src/jupyter/jupyterSettingWriter.ts
Normal file
76
extensions/notebook/src/jupyter/jupyterSettingWriter.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs-extra';
|
||||
import * as nls from 'vscode-nls';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import * as constants from '../common/constants';
|
||||
|
||||
export enum SettingType {
|
||||
String,
|
||||
Number,
|
||||
Boolean,
|
||||
Set
|
||||
}
|
||||
export class ISetting {
|
||||
key: string;
|
||||
value: string | number | boolean;
|
||||
type: SettingType;
|
||||
}
|
||||
|
||||
export class JupyterSettingWriter {
|
||||
private settings: ISetting[] = [];
|
||||
|
||||
constructor(private baseFile: string) {
|
||||
}
|
||||
|
||||
public addSetting(setting: ISetting): void {
|
||||
this.settings.push(setting);
|
||||
}
|
||||
|
||||
public async writeSettings(targetFile: string): Promise<void> {
|
||||
let settings = await this.printSettings();
|
||||
await fs.writeFile(targetFile, settings);
|
||||
}
|
||||
|
||||
public async printSettings(): Promise<string> {
|
||||
let content = '';
|
||||
let newLine = process.platform === constants.winPlatform ? '\r\n' : '\n';
|
||||
if (this.baseFile) {
|
||||
let sourceContents = await fs.readFile(this.baseFile);
|
||||
content += sourceContents.toString();
|
||||
}
|
||||
|
||||
for (let setting of this.settings) {
|
||||
content += newLine;
|
||||
content += this.printSetting(setting);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
private printSetting(setting: ISetting): string {
|
||||
let value: string;
|
||||
switch (setting.type) {
|
||||
case SettingType.Boolean:
|
||||
value = setting.value ? 'True' : 'False';
|
||||
break;
|
||||
case SettingType.String:
|
||||
value = `'${setting.value}'`;
|
||||
break;
|
||||
case SettingType.Number:
|
||||
value = `${setting.value}`;
|
||||
break;
|
||||
case SettingType.Set:
|
||||
value = `set([${setting.value}])`;
|
||||
break;
|
||||
default:
|
||||
throw new Error(localize('UnexpectedSettingType', 'Unexpected setting type {0}', setting.type));
|
||||
}
|
||||
return `c.${setting.key} = ${value}`;
|
||||
}
|
||||
}
|
||||
43
extensions/notebook/src/jupyter/remoteContentManager.ts
Normal file
43
extensions/notebook/src/jupyter/remoteContentManager.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import { nb } from 'sqlops';
|
||||
import * as vscode from 'vscode';
|
||||
import { Contents } from '@jupyterlab/services';
|
||||
|
||||
export class RemoteContentManager implements nb.ContentManager {
|
||||
|
||||
constructor(private contents: Contents.IManager) {
|
||||
}
|
||||
|
||||
public getNotebookContents(notebookUri: vscode.Uri): Thenable<nb.INotebookContents> {
|
||||
return this.getNotebookContentsAsync(notebookUri.fsPath);
|
||||
}
|
||||
|
||||
private async getNotebookContentsAsync(path: string): Promise<nb.INotebookContents> {
|
||||
if (!path) {
|
||||
return undefined;
|
||||
}
|
||||
// Note: intentionally letting caller handle exceptions
|
||||
let contentsModel = await this.contents.get(path);
|
||||
if (!contentsModel) {
|
||||
return undefined;
|
||||
}
|
||||
return <nb.INotebookContents>contentsModel.content;
|
||||
}
|
||||
|
||||
public async save(notebookUri: vscode.Uri, notebook: nb.INotebookContents): Promise<nb.INotebookContents> {
|
||||
let path = notebookUri.fsPath;
|
||||
await this.contents.save(path, {
|
||||
path: path,
|
||||
content: notebook,
|
||||
type: 'notebook',
|
||||
format: 'json'
|
||||
});
|
||||
return notebook;
|
||||
}
|
||||
}
|
||||
399
extensions/notebook/src/jupyter/serverInstance.ts
Normal file
399
extensions/notebook/src/jupyter/serverInstance.ts
Normal file
@@ -0,0 +1,399 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as UUID from 'vscode-languageclient/lib/utils/uuid';
|
||||
import * as vscode from 'vscode';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as os from 'os';
|
||||
import { spawn, ExecOptions, SpawnOptions, ChildProcess } from 'child_process';
|
||||
import * as nls from 'vscode-nls';
|
||||
const localize = nls.loadMessageBundle();
|
||||
|
||||
import { IServerInstance } from './common';
|
||||
import JupyterServerInstallation from './jupyterServerInstallation';
|
||||
import * as utils from '../common/utils';
|
||||
import * as constants from '../common/constants';
|
||||
import * as notebookUtils from '../common/notebookUtils';
|
||||
import * as ports from '../common/ports';
|
||||
|
||||
const NotebookConfigFilename = 'jupyter_notebook_config.py';
|
||||
const CustomJsFilename = 'custom.js';
|
||||
const defaultPort = 8888;
|
||||
const JupyterStartedMessage = 'The Jupyter Notebook is running';
|
||||
|
||||
type MessageListener = (data: string | Buffer) => void;
|
||||
type ErrorListener = (err: any) => void;
|
||||
|
||||
export interface IInstanceOptions {
|
||||
/**
|
||||
* The path to the initial document we want to start this server for
|
||||
*/
|
||||
documentPath: string;
|
||||
|
||||
/**
|
||||
* Base install information needed in order to start the server instance
|
||||
*/
|
||||
install: JupyterServerInstallation;
|
||||
|
||||
/**
|
||||
* Optional start directory for the notebook server. If none is set, will use a
|
||||
* path relative to the initial document
|
||||
*/
|
||||
notebookDirectory?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class to enable testing without calling into file system or
|
||||
* commandline shell APIs
|
||||
*/
|
||||
export class ServerInstanceUtils {
|
||||
public mkDir(dirPath: string, outputChannel?: vscode.OutputChannel): Promise<void> {
|
||||
return utils.mkDir(dirPath, outputChannel);
|
||||
}
|
||||
public removeDir(dirPath: string): Promise<void> {
|
||||
return fs.remove(dirPath);
|
||||
}
|
||||
public pathExists(dirPath: string): Promise<boolean> {
|
||||
return fs.pathExists(dirPath);
|
||||
}
|
||||
public copy(src: string, dest: string): Promise<void> {
|
||||
return fs.copy(src, dest);
|
||||
}
|
||||
public existsSync(dirPath: string): boolean {
|
||||
return fs.existsSync(dirPath);
|
||||
}
|
||||
public generateUuid(): string {
|
||||
return UUID.generateUuid();
|
||||
}
|
||||
public executeBufferedCommand(cmd: string, options: ExecOptions, outputChannel?: vscode.OutputChannel): Thenable<string> {
|
||||
return utils.executeBufferedCommand(cmd, options, outputChannel);
|
||||
}
|
||||
|
||||
public spawn(command: string, args?: ReadonlyArray<string>, options?: SpawnOptions): ChildProcess {
|
||||
return spawn(command, args, options);
|
||||
}
|
||||
|
||||
public checkProcessDied(childProcess: ChildProcess): void {
|
||||
if (!childProcess) {
|
||||
return;
|
||||
}
|
||||
// Wait 10 seconds and then force kill. Jupyter stop is slow so this seems a reasonable time limit
|
||||
setTimeout(() => {
|
||||
// Test if the process is still alive. Throws an exception if not
|
||||
try {
|
||||
process.kill(childProcess.pid, <any>0);
|
||||
} catch (error) {
|
||||
// All is fine.
|
||||
}
|
||||
}, 10000);
|
||||
}
|
||||
}
|
||||
|
||||
export class PerNotebookServerInstance implements IServerInstance {
|
||||
|
||||
/**
|
||||
* Root of the jupyter directory structure. Config and data roots will be
|
||||
* under this, in order to simplify deletion of folders on stop of the instance
|
||||
*/
|
||||
private baseDir: string;
|
||||
|
||||
/**
|
||||
* Path to configuration folder for this instance. Typically:
|
||||
* %extension_path%/jupyter_config/%server%_config
|
||||
*/
|
||||
private instanceConfigRoot: string;
|
||||
|
||||
/**
|
||||
* Path to data folder for this instance. Typically:
|
||||
* %extension_path%/jupyter_config/%server%_data
|
||||
*/
|
||||
private instanceDataRoot: string;
|
||||
|
||||
private _systemJupyterDir: string;
|
||||
private _port: string;
|
||||
private _uri: vscode.Uri;
|
||||
private _isStarted: boolean = false;
|
||||
private utils: ServerInstanceUtils;
|
||||
private childProcess: ChildProcess;
|
||||
private errorHandler: ErrorHandler = new ErrorHandler();
|
||||
|
||||
constructor(private options: IInstanceOptions, fsUtils?: ServerInstanceUtils) {
|
||||
this.utils = fsUtils || new ServerInstanceUtils();
|
||||
}
|
||||
|
||||
public get isStarted(): boolean {
|
||||
return this._isStarted;
|
||||
}
|
||||
|
||||
public get port(): string {
|
||||
return this._port;
|
||||
}
|
||||
|
||||
public get uri(): vscode.Uri {
|
||||
return this._uri;
|
||||
}
|
||||
|
||||
public async configure(): Promise<void> {
|
||||
await this.configureJupyter();
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
await this.startInternal();
|
||||
}
|
||||
|
||||
public async stop(): Promise<void> {
|
||||
try {
|
||||
if (this.baseDir) {
|
||||
let exists = await this.utils.pathExists(this.baseDir);
|
||||
if (exists) {
|
||||
await this.utils.removeDir(this.baseDir);
|
||||
}
|
||||
}
|
||||
if (this.isStarted) {
|
||||
let install = this.options.install;
|
||||
let stopCommand = `${install.pythonExecutable} -m jupyter notebook stop ${this._port}`;
|
||||
await this.utils.executeBufferedCommand(stopCommand, install.execOptions, install.outputChannel);
|
||||
this._isStarted = false;
|
||||
this.utils.checkProcessDied(this.childProcess);
|
||||
this.handleConnectionClosed();
|
||||
}
|
||||
} catch (error) {
|
||||
// For now, we don't care as this is non-critical
|
||||
this.notify(this.options.install, localize('serverStopError', 'Error stopping Notebook Server: {0}', utils.getErrorMessage(error)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private async configureJupyter(): Promise<void> {
|
||||
await this.createInstanceFolders();
|
||||
let resourcesFolder = path.join(this.options.install.extensionPath, 'resources', constants.jupyterConfigRootFolder);
|
||||
await this.copyInstanceConfig(resourcesFolder);
|
||||
await this.CopyCustomJs(resourcesFolder);
|
||||
await this.copyKernelsToSystemJupyterDirs();
|
||||
}
|
||||
|
||||
private async createInstanceFolders(): Promise<void> {
|
||||
this.baseDir = path.join(this.options.install.configRoot, 'instances', `${this.utils.generateUuid()}`);
|
||||
this.instanceConfigRoot = path.join(this.baseDir, 'config');
|
||||
this.instanceDataRoot = path.join(this.baseDir, 'data');
|
||||
await this.utils.mkDir(this.baseDir, this.options.install.outputChannel);
|
||||
await this.utils.mkDir(this.instanceConfigRoot, this.options.install.outputChannel);
|
||||
await this.utils.mkDir(this.instanceDataRoot, this.options.install.outputChannel);
|
||||
}
|
||||
|
||||
private async copyInstanceConfig(resourcesFolder: string): Promise<void> {
|
||||
let configSource = path.join(resourcesFolder, NotebookConfigFilename);
|
||||
let configDest = path.join(this.instanceConfigRoot, NotebookConfigFilename);
|
||||
await this.utils.copy(configSource, configDest);
|
||||
}
|
||||
|
||||
private async CopyCustomJs(resourcesFolder: string): Promise<void> {
|
||||
let customPath = path.join(this.instanceConfigRoot, 'custom');
|
||||
await this.utils.mkDir(customPath, this.options.install.outputChannel);
|
||||
let customSource = path.join(resourcesFolder, CustomJsFilename);
|
||||
let customDest = path.join(customPath, CustomJsFilename);
|
||||
await this.utils.copy(customSource, customDest);
|
||||
}
|
||||
|
||||
private async copyKernelsToSystemJupyterDirs(): Promise<void> {
|
||||
let kernelsExtensionSource = path.join(this.options.install.extensionPath, 'kernels');
|
||||
this._systemJupyterDir = this.getSystemJupyterKernelDir();
|
||||
if (!this.utils.existsSync(this._systemJupyterDir)) {
|
||||
await this.utils.mkDir(this._systemJupyterDir, this.options.install.outputChannel);
|
||||
}
|
||||
await this.utils.copy(kernelsExtensionSource, this._systemJupyterDir);
|
||||
}
|
||||
|
||||
private getSystemJupyterKernelDir(): string {
|
||||
switch (process.platform) {
|
||||
case 'win32':
|
||||
let appDataWindows = process.env['APPDATA'];
|
||||
return appDataWindows + '\\jupyter\\kernels';
|
||||
case 'darwin':
|
||||
return path.resolve(os.homedir(), 'Library/Jupyter/kernels');
|
||||
default:
|
||||
return path.resolve(os.homedir(), '.local/share/jupyter/kernels');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Starts a Jupyter instance using the provided a start command. Server is determined to have
|
||||
* started when the log message with URL to connect to is emitted.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
protected async startInternal(): Promise<void> {
|
||||
if (this.isStarted) {
|
||||
return;
|
||||
}
|
||||
let notebookDirectory = this.getNotebookDirectory();
|
||||
// Find a port in a given range. If run into trouble, got up 100 in range and search inside a larger range
|
||||
let port = await ports.strictFindFreePort(new ports.StrictPortFindOptions(defaultPort, defaultPort + 100, defaultPort + 1000));
|
||||
let token = await notebookUtils.getRandomToken();
|
||||
this._uri = vscode.Uri.parse(`http://localhost:${port}/?token=${token}`);
|
||||
this._port = port.toString();
|
||||
let startCommand = `${this.options.install.pythonExecutable} -m jupyter notebook --no-browser --notebook-dir "${notebookDirectory}" --port=${port} --NotebookApp.token=${token}`;
|
||||
this.notifyStarting(this.options.install, startCommand);
|
||||
|
||||
// Execute the command
|
||||
await this.executeStartCommand(startCommand);
|
||||
}
|
||||
|
||||
private executeStartCommand(startCommand: string): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let install = this.options.install;
|
||||
this.childProcess = this.spawnJupyterProcess(install, startCommand);
|
||||
|
||||
// Add listeners for the process exiting prematurely
|
||||
let onErrorBeforeStartup = (err) => reject(err);
|
||||
let onExitBeforeStart = (err) => {
|
||||
if (!this.isStarted) {
|
||||
reject(localize('notebookStartProcessExitPremature', 'Notebook process exited prematurely with error: {0}', err));
|
||||
}
|
||||
};
|
||||
this.childProcess.on('error', onErrorBeforeStartup);
|
||||
this.childProcess.on('exit', onExitBeforeStart);
|
||||
|
||||
// Add listener for the process to emit its web address
|
||||
let handleStdout = (data: string | Buffer) => { install.outputChannel.appendLine(data.toString()); };
|
||||
let handleStdErr = (data: string | Buffer) => {
|
||||
// For some reason, URL info is sent on StdErr
|
||||
let [url, port] = this.matchUrlAndPort(data);
|
||||
if (url) {
|
||||
// For now, will verify port matches
|
||||
if (url.authority !== this._uri.authority
|
||||
|| url.query !== this._uri.query) {
|
||||
this._uri = url;
|
||||
this._port = port;
|
||||
}
|
||||
this.notifyStarted(install, url.toString());
|
||||
this._isStarted = true;
|
||||
|
||||
this.updateListeners(handleStdout, handleStdErr, onErrorBeforeStartup, onExitBeforeStart);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
this.childProcess.stdout.on('data', handleStdout);
|
||||
this.childProcess.stderr.on('data', handleStdErr);
|
||||
});
|
||||
}
|
||||
|
||||
private updateListeners(handleStdout: MessageListener, handleStdErr: MessageListener, onErrorBeforeStartup: ErrorListener, onExitBeforeStart: ErrorListener): void {
|
||||
this.childProcess.stdout.removeListener('data', handleStdout);
|
||||
this.childProcess.stderr.removeListener('data', handleStdErr);
|
||||
this.childProcess.removeListener('error', onErrorBeforeStartup);
|
||||
this.childProcess.removeListener('exit', onExitBeforeStart);
|
||||
|
||||
this.childProcess.addListener('error', this.handleConnectionError);
|
||||
this.childProcess.addListener('exit', this.handleConnectionClosed);
|
||||
|
||||
// TODO #897 covers serializing stdout and stderr to a location where we can read from so that user can see if they run into trouble
|
||||
}
|
||||
|
||||
private handleConnectionError(error: Error): void {
|
||||
let action = this.errorHandler.handleError(error);
|
||||
if (action === ErrorAction.Shutdown) {
|
||||
this.notify(this.options.install, localize('jupyterError', 'Error sent from Jupyter: {0}', utils.getErrorMessage(error)));
|
||||
this.stop();
|
||||
}
|
||||
}
|
||||
private handleConnectionClosed(): void {
|
||||
this.childProcess = undefined;
|
||||
this._isStarted = false;
|
||||
}
|
||||
|
||||
getNotebookDirectory(): string {
|
||||
if (this.options.notebookDirectory) {
|
||||
if (this.options.notebookDirectory.endsWith('\\')) {
|
||||
return this.options.notebookDirectory.substr(0, this.options.notebookDirectory.length - 1) + '/';
|
||||
}
|
||||
return this.options.notebookDirectory;
|
||||
}
|
||||
return path.dirname(this.options.documentPath);
|
||||
}
|
||||
|
||||
private matchUrlAndPort(data: string | Buffer): [vscode.Uri, string] {
|
||||
// regex: Looks for the successful startup log message like:
|
||||
// [C 12:08:51.947 NotebookApp]
|
||||
//
|
||||
// Copy/paste this URL into your browser when you connect for the first time,
|
||||
// to login with a token:
|
||||
// http://localhost:8888/?token=f5ee846e9bd61c3a8d835ecd9b965591511a331417b997b7
|
||||
let dataString = data.toString();
|
||||
let urlMatch = dataString.match(/\[C[\s\S]+ {8}(.+:(\d+)\/.*)$/m);
|
||||
|
||||
if (urlMatch) {
|
||||
// Legacy case: manually parse token info if no token/port were passed
|
||||
return [vscode.Uri.parse(urlMatch[1]), urlMatch[2]];
|
||||
} else if (this._uri && dataString.indexOf(JupyterStartedMessage) > -1) {
|
||||
// Default case: detect the notebook started message, indicating our preferred port and token were used
|
||||
return [this._uri, this._port];
|
||||
}
|
||||
return [undefined, undefined];
|
||||
}
|
||||
|
||||
private notifyStarted(install: JupyterServerInstallation, jupyterUri: string): void {
|
||||
install.outputChannel.appendLine(localize('jupyterOutputMsgStartSuccessful', '... Jupyter is running at {0}', jupyterUri));
|
||||
}
|
||||
private notify(install: JupyterServerInstallation, message: string): void {
|
||||
install.outputChannel.appendLine(message);
|
||||
}
|
||||
|
||||
private notifyStarting(install: JupyterServerInstallation, startCommand: string): void {
|
||||
install.outputChannel.appendLine(localize('jupyterOutputMsgStart', '... Starting Notebook server'));
|
||||
install.outputChannel.appendLine(` > ${startCommand}`);
|
||||
}
|
||||
|
||||
private spawnJupyterProcess(install: JupyterServerInstallation, startCommand: string): ChildProcess {
|
||||
// Specify the global environment variables
|
||||
let env = this.getEnvWithConfigPaths();
|
||||
// Setting the PATH variable here for the jupyter command. Apparently setting it above will cause the
|
||||
// notebook process to die even though we don't override it with the for loop logic above.
|
||||
delete env['Path'];
|
||||
env['PATH'] = install.pythonEnvVarPath;
|
||||
|
||||
// 'MSHOST_TELEMETRY_ENABLED' and 'MSHOST_ENVIRONMENT' environment variables are set
|
||||
// for telemetry purposes used by PROSE in the process where the Jupyter kernel runs
|
||||
if (vscode.workspace.getConfiguration('telemetry').get<boolean>('enableTelemetry', true)) {
|
||||
env['MSHOST_TELEMETRY_ENABLED'] = true;
|
||||
} else {
|
||||
env['MSHOST_TELEMETRY_ENABLED'] = false;
|
||||
}
|
||||
|
||||
env['MSHOST_ENVIRONMENT'] = 'ADSClient-' + vscode.version;
|
||||
|
||||
// Start the notebook process
|
||||
let options = {
|
||||
shell: true,
|
||||
env: env
|
||||
};
|
||||
let childProcess = this.utils.spawn(startCommand, [], options);
|
||||
return childProcess;
|
||||
}
|
||||
|
||||
private getEnvWithConfigPaths(): any {
|
||||
let env = Object.assign({}, process.env);
|
||||
env['JUPYTER_CONFIG_DIR'] = this.instanceConfigRoot;
|
||||
env['JUPYTER_PATH'] = this.instanceDataRoot;
|
||||
return env;
|
||||
}
|
||||
}
|
||||
|
||||
class ErrorHandler {
|
||||
private numErrors: number = 0;
|
||||
|
||||
public handleError(error: Error): ErrorAction {
|
||||
this.numErrors++;
|
||||
return this.numErrors > 3 ? ErrorAction.Shutdown : ErrorAction.Continue;
|
||||
}
|
||||
}
|
||||
|
||||
enum ErrorAction {
|
||||
Continue = 1,
|
||||
Shutdown = 2
|
||||
}
|
||||
Reference in New Issue
Block a user