ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Port most notebook model code over to be behind a service (#3068)

Browse Source 
- Defines a new NotebookService in Azure Data Studio which will be used to interact with notebooks. Since notebooks can require per-file instantiation the provider is just used to create & track managers for a given URI.
- Inject this into notebook.component.ts and pass required parameters that'll be used to properly initialize a manger into the method. Actual initialization not done yet.
- Port over & recompile notebook model code
- Define most required APIs in sqlops.proposed.d.ts. In the future, these will be used by extensions to contribute their own providers.
This commit is contained in:
Kevin Cunnane
2018-10-31 22:01:40 -07:00
committed by GitHub
parent ac0ffab99c
commit fc3bf45a7f
29 changed files with 2973 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

321
src/sql/parts/notebook/models/cell.ts Normal file
View File

@@ -0,0 +1,321 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { Event, Emitter } from 'vs/base/common/event';
import URI from 'vs/base/common/uri';
import { nb } from 'sqlops';
import { ICellModelOptions, IModelFactory } from './modelInterfaces';
import * as notebookUtils from '../notebookUtils';
import { CellTypes, CellType, NotebookChangeType } from 'sql/parts/notebook/models/contracts';
import { ICellModel } from 'sql/parts/notebook/models/modelInterfaces';
let modelId = 0;
export class CellModel implements ICellModel {
private static LanguageMapping: Map<string, string>;
private _cellType: nb.CellType;
private _source: string;
private _language: string;
private _future: nb.IFuture;
private _outputs: nb.ICellOutput[] = [];
private _isEditMode: boolean;
private _onOutputsChanged = new Emitter<ReadonlyArray<nb.ICellOutput>>();
private _onCellModeChanged = new Emitter<boolean>();
public id: string;
private _isTrusted: boolean;
private _active: boolean;
private _cellUri: URI;
constructor(private factory: IModelFactory, cellData?: nb.ICell, private _options?: ICellModelOptions) {
this.id = `${modelId++}`;
CellModel.CreateLanguageMappings();
// Do nothing for now
if (cellData) {
this.fromJSON(cellData);
} else {
this._cellType = CellTypes.Code;
this._source = '';
}
this._isEditMode = this._cellType !== CellTypes.Markdown;
this.setDefaultLanguage();
if (_options && _options.isTrusted) {
this._isTrusted = true;
} else {
this._isTrusted = false;
}
}
public equals(other: ICellModel) {
return other && other.id === this.id;
}
public get onOutputsChanged(): Event<ReadonlyArray<nb.ICellOutput>> {
return this._onOutputsChanged.event;
}
public get onCellModeChanged(): Event<boolean> {
return this._onCellModeChanged.event;
}
public get isEditMode(): boolean {
return this._isEditMode;
}
public get future(): nb.IFuture {
return this._future;
}
public set isEditMode(isEditMode: boolean) {
this._isEditMode = isEditMode;
this._onCellModeChanged.fire(this._isEditMode);
// Note: this does not require a notebook update as it does not change overall state
}
public get trustedMode(): boolean {
return this._isTrusted;
}
public set trustedMode(isTrusted: boolean) {
if (this._isTrusted !== isTrusted) {
this._isTrusted = isTrusted;
this._onOutputsChanged.fire(this._outputs);
}
}
public get active(): boolean {
return this._active;
}
public set active(value: boolean) {
this._active = value;
}
public get cellUri(): URI {
return this._cellUri;
}
public set cellUri(value: URI) {
this._cellUri = value;
}
public get options(): ICellModelOptions {
return this._options;
}
public get cellType(): CellType {
return this._cellType;
}
public get source(): string {
return this._source;
}
public set source(newSource: string) {
if (this._source !== newSource) {
this._source = newSource;
this.sendChangeToNotebook(NotebookChangeType.CellSourceUpdated);
}
}
public get language(): string {
return this._language;
}
public set language(newLanguage: string) {
this._language = newLanguage;
}
/**
* Sets the future which will be used to update the output
* area for this cell
*/
setFuture(future: nb.IFuture): void {
if (this._future === future) {
// Nothing to do
return;
}
// Setting the future indicates the cell is running which enables trusted mode.
// See https://jupyter-notebook.readthedocs.io/en/stable/security.html
this._isTrusted = true;
if (this._future) {
this._future.dispose();
}
this.clearOutputs();
this._future = future;
future.setReplyHandler({ handle: (msg) => this.handleReply(msg) });
future.setIOPubHandler({ handle: (msg) => this.handleIOPub(msg) });
}
private clearOutputs(): void {
this._outputs = [];
this.fireOutputsChanged();
}
private fireOutputsChanged(): void {
this._onOutputsChanged.fire(this.outputs);
this.sendChangeToNotebook(NotebookChangeType.CellOutputUpdated);
}
private sendChangeToNotebook(change: NotebookChangeType): void {
if (this._options && this._options.notebook) {
this._options.notebook.onCellChange(this, change);
}
}
public get outputs(): ReadonlyArray<nb.ICellOutput> {
return this._outputs;
}
private handleReply(msg: nb.IShellMessage): void {
// TODO #931 we should process this. There can be a payload attached which should be added to outputs.
// In all other cases, it is a no-op
let output: nb.ICellOutput = msg.content as nb.ICellOutput;
}
private handleIOPub(msg: nb.IIOPubMessage): void {
let msgType = msg.header.msg_type;
let displayId = this.getDisplayId(msg);
let output: nb.ICellOutput;
switch (msgType) {
case 'execute_result':
case 'display_data':
case 'stream':
case 'error':
output = msg.content as nb.ICellOutput;
output.output_type = msgType;
break;
case 'clear_output':
// TODO wait until next message before clearing
// let wait = (msg as KernelMessage.IClearOutputMsg).content.wait;
this.clearOutputs();
break;
case 'update_display_data':
output = msg.content as nb.ICellOutput;
output.output_type = 'display_data';
// TODO #930 handle in-place update of displayed data
// targets = this._displayIdMap.get(displayId);
// if (targets) {
// for (let index of targets) {
// model.set(index, output);
// }
// }
break;
default:
break;
}
// TODO handle in-place update of displayed data
// if (displayId && msgType === 'display_data') {
// targets = this._displayIdMap.get(displayId) || [];
// targets.push(model.length - 1);
// this._displayIdMap.set(displayId, targets);
// }
if (output) {
this._outputs.push(output);
this.fireOutputsChanged();
}
}
private getDisplayId(msg: nb.IIOPubMessage): string | undefined {
let transient = (msg.content.transient || {});
return transient['display_id'] as string;
}
public toJSON(): nb.ICell {
let cellJson: Partial<nb.ICell> = {
cell_type: this._cellType,
source: this._source,
metadata: {
}
};
if (this._cellType === CellTypes.Code) {
cellJson.metadata.language = this._language,
cellJson.outputs = this._outputs;
cellJson.execution_count = 1; // TODO: keep track of actual execution count
}
return cellJson as nb.ICell;
}
public fromJSON(cell: nb.ICell): void {
if (!cell) {
return;
}
this._cellType = cell.cell_type;
this._source = Array.isArray(cell.source) ? cell.source.join('') : cell.source;
this._language = (cell.metadata && cell.metadata.language) ? cell.metadata.language : 'python';
if (cell.outputs) {
for (let output of cell.outputs) {
// For now, we're assuming it's OK to save these as-is with no modification
this.addOutput(output);
}
}
}
private addOutput(output: nb.ICellOutput) {
this._normalize(output);
this._outputs.push(output);
}
/**
* Normalize an output.
*/
private _normalize(value: nb.ICellOutput): void {
if (notebookUtils.isStream(value)) {
if (Array.isArray(value.text)) {
value.text = (value.text as string[]).join('\n');
}
}
}
private static CreateLanguageMappings(): void {
if (CellModel.LanguageMapping) {
return;
}
CellModel.LanguageMapping = new Map<string, string>();
CellModel.LanguageMapping['pyspark'] = 'python';
CellModel.LanguageMapping['pyspark3'] = 'python';
CellModel.LanguageMapping['python'] = 'python';
CellModel.LanguageMapping['scala'] = 'scala';
}
private get languageInfo(): nb.ILanguageInfo {
if (this._options && this._options.notebook && this._options.notebook.languageInfo) {
return this._options.notebook.languageInfo;
}
return undefined;
}
private setDefaultLanguage(): void {
this._language = 'python';
// In languageInfo, set the language to the "name" property
// If the "name" property isn't defined, check the "mimeType" property
// Otherwise, default to python as the language
let languageInfo = this.languageInfo;
if (languageInfo) {
if (languageInfo.name) {
// check the LanguageMapping to determine if a mapping is necessary (example 'pyspark' -> 'python')
if (CellModel.LanguageMapping[languageInfo.name]) {
this._language = CellModel.LanguageMapping[languageInfo.name];
} else {
this._language = languageInfo.name;
}
} else if (languageInfo.mimetype) {
this._language = languageInfo.mimetype;
}
}
let mimeTypePrefix = 'x-';
if (this._language.includes(mimeTypePrefix)) {
this._language = this._language.replace(mimeTypePrefix, '');
}
}
}

360
src/sql/parts/notebook/models/clientSession.ts Normal file
View File

@@ -0,0 +1,360 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
// This code is based on @jupyterlab/packages/apputils/src/clientsession.tsx
'use strict';
import { nb } from 'sqlops';
import * as nls from 'vs/nls';
import URI from 'vs/base/common/uri';
import { Event, Emitter } from 'vs/base/common/event';
import { IClientSession, IKernelPreference, IClientSessionOptions } from './modelInterfaces';
import { Deferred } from 'sql/base/common/promise';
import * as notebookUtils from '../notebookUtils';
import * as sparkUtils from '../spark/sparkUtils';
import { INotebookManager } from 'sql/services/notebook/notebookService';
import { IConnectionProfile } from 'sql/parts/connection/common/interfaces';
import { NotebookConnection } from 'sql/parts/notebook/models/notebookConnection';
/**
* Implementation of a client session. This is a model over session operations,
* which may come from the session manager or a specific session.
*/
export class ClientSession implements IClientSession {
//#region private fields with public accessors
private _terminatedEmitter = new Emitter<void>();
private _kernelChangedEmitter = new Emitter<nb.IKernelChangedArgs>();
private _statusChangedEmitter = new Emitter<nb.ISession>();
private _iopubMessageEmitter = new Emitter<nb.IMessage>();
private _unhandledMessageEmitter = new Emitter<nb.IMessage>();
private _propertyChangedEmitter = new Emitter<'path' | 'name' | 'type'>();
private _path: string;
private _type: string;
private _name: string;
private _isReady: boolean;
private _ready: Deferred<void>;
private _kernelChangeCompleted: Deferred<void>;
private _kernelPreference: IKernelPreference;
private _kernelDisplayName: string;
private _errorMessage: string;
//#endregion
private _serverLoadFinished: Promise<void>;
private _session: nb.ISession;
private isServerStarted: boolean;
private notebookManager: INotebookManager;
private _connection: NotebookConnection;
private _kernelConfigActions: ((kernelName: string) => Promise<any>)[] = [];
constructor(private options: IClientSessionOptions) {
this._path = options.path;
this.notebookManager = options.notebookManager;
this._isReady = false;
this._ready = new Deferred<void>();
this._kernelChangeCompleted = new Deferred<void>();
}
public async initialize(connection?: NotebookConnection): Promise<void> {
try {
this._kernelConfigActions.push((kernelName: string) => { return this.runTasksBeforeSessionStart(kernelName); });
this._connection = connection;
this._serverLoadFinished = this.startServer();
await this._serverLoadFinished;
await this.initializeSession();
} catch (err) {
this._errorMessage = notebookUtils.getErrorMessage(err);
}
// Always resolving for now. It's up to callers to check for error case
this._isReady = true;
this._ready.resolve();
this._kernelChangeCompleted.resolve();
}
private async startServer(): Promise<void> {
let serverManager = this.notebookManager.serverManager;
if (serverManager && !serverManager.isStarted) {
await serverManager.startServer();
if (!serverManager.isStarted) {
throw new Error(nls.localize('ServerNotStarted', 'Server did not start for unknown reason'));
}
this.isServerStarted = serverManager.isStarted;
} else {
this.isServerStarted = true;
}
}
private async initializeSession(): Promise<void> {
await this._serverLoadFinished;
if (this.isServerStarted) {
if (!this.notebookManager.sessionManager.isReady) {
await this.notebookManager.sessionManager.ready;
}
if (this._kernelPreference && this._kernelPreference.shouldStart) {
await this.startSessionInstance(this._kernelPreference.name);
}
}
}
private async startSessionInstance(kernelName: string): Promise<void> {
let session: nb.ISession;
try {
session = await this.notebookManager.sessionManager.startNew({
path: this.path,
kernelName: kernelName
// TODO add kernel name if saved in the document
});
session.defaultKernelLoaded = true;
} catch (err) {
// TODO move registration
if (err && err.response && err.response.status === 501) {
this.options.notificationService.warn(nls.localize('sparkKernelRequiresConnection', 'Kernel {0} was not found. The default kernel will be used instead.', kernelName));
session = await this.notebookManager.sessionManager.startNew({
path: this.path,
kernelName: undefined
});
} else {
throw err;
}
session.defaultKernelLoaded = false;
}
this._session = session;
await this.runKernelConfigActions(kernelName);
this._statusChangedEmitter.fire(session);
}
private async runKernelConfigActions(kernelName: string): Promise<void> {
for (let startAction of this._kernelConfigActions) {
await startAction(kernelName);
}
}
public dispose(): void {
// No-op for now
}
/**
* Indicates the server has finished loading. It may have failed to load in
* which case the view will be in an error state.
*/
public get serverLoadFinished(): Promise<void> {
return this._serverLoadFinished;
}
//#region IClientSession Properties
public get terminated(): Event<void> {
return this._terminatedEmitter.event;
}
public get kernelChanged(): Event<nb.IKernelChangedArgs> {
return this._kernelChangedEmitter.event;
}
public get statusChanged(): Event<nb.ISession> {
return this._statusChangedEmitter.event;
}
public get iopubMessage(): Event<nb.IMessage> {
return this._iopubMessageEmitter.event;
}
public get unhandledMessage(): Event<nb.IMessage> {
return this._unhandledMessageEmitter.event;
}
public get propertyChanged(): Event<'path' | 'name' | 'type'> {
return this._propertyChangedEmitter.event;
}
public get kernel(): nb.IKernel | null {
return this._session ? this._session.kernel : undefined;
}
public get path(): string {
return this._path;
}
public get name(): string {
return this._name;
}
public get type(): string {
return this._type;
}
public get status(): nb.KernelStatus {
if (!this.isReady) {
return 'starting';
}
return this._session ? this._session.status : 'dead';
}
public get isReady(): boolean {
return this._isReady;
}
public get ready(): Promise<void> {
return this._ready.promise;
}
public get kernelChangeCompleted(): Promise<void> {
return this._kernelChangeCompleted.promise;
}
public get kernelPreference(): IKernelPreference {
return this._kernelPreference;
}
public set kernelPreference(value: IKernelPreference) {
this._kernelPreference = value;
}
public get kernelDisplayName(): string {
return this._kernelDisplayName;
}
public get errorMessage(): string {
return this._errorMessage;
}
public get isInErrorState(): boolean {
return !!this._errorMessage;
}
//#endregion
//#region Not Yet Implemented
/**
* Change the current kernel associated with the document.
*/
async changeKernel(options: nb.IKernelSpec): Promise<nb.IKernel> {
this._kernelChangeCompleted = new Deferred<void>();
this._isReady = false;
let oldKernel = this.kernel;
let newKernel = this.kernel;
let kernel = await this.doChangeKernel(options);
try {
await kernel.ready;
} catch (error) {
// Cleanup some state before re-throwing
this._isReady = kernel.isReady;
this._kernelChangeCompleted.resolve();
throw error;
}
newKernel = this._session ? kernel : this._session.kernel;
this._isReady = kernel.isReady;
// Send resolution events to listeners
this._kernelChangeCompleted.resolve();
this._kernelChangedEmitter.fire({
oldValue: oldKernel,
newValue: newKernel
});
return kernel;
}
/**
* Helper method to either call ChangeKernel on current session, or start a new session
* @param options
*/
private async doChangeKernel(options: nb.IKernelSpec): Promise<nb.IKernel> {
let kernel: nb.IKernel;
if (this._session) {
kernel = await this._session.changeKernel(options);
await this.runKernelConfigActions(kernel.name);
} else {
kernel = await this.startSessionInstance(options.name).then(() => this.kernel);
}
return kernel;
}
public async runTasksBeforeSessionStart(kernelName: string): Promise<void> {
// TODO we should move all Spark-related code to SparkMagicContext
if (this._session && this._connection && this.isSparkKernel(kernelName)) {
// TODO may need to reenable a way to get the credential
// await this._connection.getCredential();
// %_do_not_call_change_endpoint is a SparkMagic command that lets users change endpoint options,
// such as user/profile/host name/auth type
let server = URI.parse(sparkUtils.getLivyUrl(this._connection.host, this._connection.knoxport)).toString();
let doNotCallChangeEndpointParams =
`%_do_not_call_change_endpoint --username=${this._connection.user} --password=${this._connection.password} --server=${server} --auth=Basic_Access`;
let future = this._session.kernel.requestExecute({
code: doNotCallChangeEndpointParams
}, true);
await future.done;
}
}
public async updateConnection(connection: NotebookConnection): Promise<void> {
if (!this.kernel) {
// TODO is there any case where skipping causes errors? Do far it seems like it gets called twice
return;
}
this._connection = (connection.connectionProfile.id !== '-1') ? connection : this._connection;
// if kernel is not set, don't run kernel config actions
// this should only occur when a cell is cancelled, which interrupts the kernel
if (this.kernel && this.kernel.name) {
await this.runKernelConfigActions(this.kernel.name);
}
}
isSparkKernel(kernelName: string): any {
return kernelName && kernelName.toLowerCase().indexOf('spark') > -1;
}
/**
* Kill the kernel and shutdown the session.
*
* @returns A promise that resolves when the session is shut down.
*/
public async shutdown(): Promise<void> {
// Always try to shut down session
if (this._session && this._session.id) {
this.notebookManager.sessionManager.shutdown(this._session.id);
}
let serverManager = this.notebookManager.serverManager;
if (serverManager) {
await serverManager.stopServer();
}
}
/**
* Select a kernel for the session.
*/
selectKernel(): Promise<void> {
throw new Error('Not implemented');
}
/**
* Restart the session.
*
* @returns A promise that resolves with whether the kernel has restarted.
*
* #### Notes
* If there is a running kernel, present a dialog.
* If there is no kernel, we start a kernel with the last run
* kernel name and resolves with `true`. If no kernel has been started,
* this is a no-op, and resolves with `false`.
*/
restart(): Promise<boolean> {
throw new Error('Not implemented');
}
/**
* Change the session path.
*
* @param path - The new session path.
*
* @returns A promise that resolves when the session has renamed.
*
* #### Notes
* This uses the Jupyter REST API, and the response is validated.
* The promise is fulfilled on a valid response and rejected otherwise.
*/
setPath(path: string): Promise<void> {
throw new Error('Not implemented');
}
/**
* Change the session name.
*/
setName(name: string): Promise<void> {
throw new Error('Not implemented');
}
/**
* Change the session type.
*/
setType(type: string): Promise<void> {
throw new Error('Not implemented');
}
//#endregion
}

47
src/sql/parts/notebook/models/contracts.ts Normal file
View File

@@ -0,0 +1,47 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
export type CellType = 'code' | 'markdown' | 'raw';
export class CellTypes {
public static readonly Code = 'code';
public static readonly Markdown = 'markdown';
public static readonly Raw = 'raw';
}
// to do: add all mime types
export type MimeType = 'text/plain' | 'text/html';
// to do: add all mime types
export class MimeTypes {
public static readonly PlainText = 'text/plain';
public static readonly HTML = 'text/html';
}
export type OutputType =
| 'execute_result'
| 'display_data'
| 'stream'
| 'error'
| 'update_display_data';
export class OutputTypes {
public static readonly ExecuteResult = 'execute_result';
public static readonly DisplayData = 'display_data';
public static readonly Stream = 'stream';
public static readonly Error = 'error';
public static readonly UpdateDisplayData = 'update_display_data';
}
export enum NotebookChangeType {
CellsAdded,
CellDeleted,
CellSourceUpdated,
CellOutputUpdated,
DirtyStateChanged
}

23
src/sql/parts/notebook/models/modelFactory.ts Normal file
View File

@@ -0,0 +1,23 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import { CellModel } from './cell';
import { IClientSession, IClientSessionOptions, ICellModelOptions, ICellModel, IModelFactory } from './modelInterfaces';
import { ClientSession } from './clientSession';
export class ModelFactory implements IModelFactory {
public createCell(cell: nb.ICell, options: ICellModelOptions): ICellModel {
return new CellModel(this, cell, options);
}
public createClientSession(options: IClientSessionOptions): IClientSession {
return new ClientSession(options);
}
}

372
src/sql/parts/notebook/models/modelInterfaces.ts Normal file
View File

@@ -0,0 +1,372 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
// This code is based on @jupyterlab/packages/apputils/src/clientsession.tsx
'use strict';
import { nb } from 'sqlops';
import { Event } from 'vs/base/common/event';
import { IDisposable } from 'vs/base/common/lifecycle';
import URI from 'vs/base/common/uri';
import { INotificationService } from 'vs/platform/notification/common/notification';
import { CellType, NotebookChangeType } from 'sql/parts/notebook/models/contracts';
import { INotebookManager } from 'sql/services/notebook/notebookService';
import { IConnectionProfile } from 'sql/parts/connection/common/interfaces';
import { NotebookConnection } from 'sql/parts/notebook/models/notebookConnection';
import { IConnectionManagementService } from 'sql/parts/connection/common/connectionManagement';
export interface IClientSessionOptions {
path: string;
notebookManager: INotebookManager;
notificationService: INotificationService;
}
/**
* The interface of client session object.
*
* The client session represents the link between
* a path and its kernel for the duration of the lifetime
* of the session object. The session can have no current
* kernel, and can start a new kernel at any time.
*/
export interface IClientSession extends IDisposable {
/**
* A signal emitted when the session is shut down.
*/
readonly terminated: Event<void>;
/**
* A signal emitted when the kernel changes.
*/
readonly kernelChanged: Event<nb.IKernelChangedArgs>;
/**
* A signal emitted when the kernel status changes.
*/
readonly statusChanged: Event<nb.ISession>;
/**
* A signal emitted for a kernel messages.
*/
readonly iopubMessage: Event<nb.IMessage>;
/**
* A signal emitted for an unhandled kernel message.
*/
readonly unhandledMessage: Event<nb.IMessage>;
/**
* A signal emitted when a session property changes.
*/
readonly propertyChanged: Event<'path' | 'name' | 'type'>;
/**
* The current kernel associated with the document.
*/
readonly kernel: nb.IKernel | null;
/**
* The current path associated with the client session.
*/
readonly path: string;
/**
* The current name associated with the client session.
*/
readonly name: string;
/**
* The type of the client session.
*/
readonly type: string;
/**
* The current status of the client session.
*/
readonly status: nb.KernelStatus;
/**
* Whether the session is ready.
*/
readonly isReady: boolean;
/**
* Whether the session is in an unusable state
*/
readonly isInErrorState: boolean;
/**
* The error information, if this session is in an error state
*/
readonly errorMessage: string;
/**
* A promise that is fulfilled when the session is ready.
*/
readonly ready: Promise<void>;
/**
* A promise that is fulfilled when the session completes a kernel change.
*/
readonly kernelChangeCompleted: Promise<void>;
/**
* The kernel preference.
*/
kernelPreference: IKernelPreference;
/**
* The display name of the kernel.
*/
readonly kernelDisplayName: string;
/**
* Initializes the ClientSession, by starting the server and
* connecting to the SessionManager.
* This will optionally start a session if the kernel preferences
* indicate this is desired
*/
initialize(connection?: NotebookConnection): Promise<void>;
/**
* Change the current kernel associated with the document.
*/
changeKernel(
options: nb.IKernelSpec
): Promise<nb.IKernel>;
/**
* Kill the kernel and shutdown the session.
*
* @returns A promise that resolves when the session is shut down.
*/
shutdown(): Promise<void>;
/**
* Select a kernel for the session.
*/
selectKernel(): Promise<void>;
/**
* Restart the session.
*
* @returns A promise that resolves with whether the kernel has restarted.
*
* #### Notes
* If there is a running kernel, present a dialog.
* If there is no kernel, we start a kernel with the last run
* kernel name and resolves with `true`. If no kernel has been started,
* this is a no-op, and resolves with `false`.
*/
restart(): Promise<boolean>;
/**
* Change the session path.
*
* @param path - The new session path.
*
* @returns A promise that resolves when the session has renamed.
*
* #### Notes
* This uses the Jupyter REST API, and the response is validated.
* The promise is fulfilled on a valid response and rejected otherwise.
*/
setPath(path: string): Promise<void>;
/**
* Change the session name.
*/
setName(name: string): Promise<void>;
/**
* Change the session type.
*/
setType(type: string): Promise<void>;
/**
* Updates the connection
*/
updateConnection(connection: NotebookConnection): void;
}
export interface IDefaultConnection {
defaultConnection: IConnectionProfile;
otherConnections: IConnectionProfile[];
}
/**
* A kernel preference.
*/
export interface IKernelPreference {
/**
* The name of the kernel.
*/
readonly name?: string;
/**
* The preferred kernel language.
*/
readonly language?: string;
/**
* The id of an existing kernel.
*/
readonly id?: string;
/**
* Whether to prefer starting a kernel.
*/
readonly shouldStart?: boolean;
/**
* Whether a kernel can be started.
*/
readonly canStart?: boolean;
/**
* Whether to auto-start the default kernel if no matching kernel is found.
*/
readonly autoStartDefault?: boolean;
}
export interface INotebookModel {
/**
* Client Session in the notebook, used for sending requests to the notebook service
*/
readonly clientSession: IClientSession;
/**
* LanguageInfo saved in the query book
*/
readonly languageInfo: nb.ILanguageInfo;
/**
* The notebook service used to call backend APIs
*/
readonly notebookManager: INotebookManager;
/**
* Event fired on first initialization of the kernel and
* on subsequent change events
*/
readonly kernelChanged: Event<nb.IKernelChangedArgs>;
/**
* Event fired on first initialization of the kernels and
* on subsequent change events
*/
readonly kernelsChanged: Event<nb.IKernelSpec>;
/**
* Default kernel
*/
defaultKernel?: nb.IKernelSpec;
/**
* Event fired on first initialization of the contexts and
* on subsequent change events
*/
readonly contextsChanged: Event<void>;
/**
* The specs for available kernels, or undefined if these have
* not been loaded yet
*/
readonly specs: nb.IAllKernels | undefined;
/**
* The specs for available contexts, or undefined if these have
* not been loaded yet
*/
readonly contexts: IDefaultConnection | undefined;
/**
* The trusted mode of the NoteBook
*/
trustedMode: boolean;
/**
* Change the current kernel from the Kernel dropdown
* @param displayName kernel name (as displayed in Kernel dropdown)
*/
changeKernel(displayName: string): void;
/**
* Change the current context (if applicable)
*/
changeContext(host: string): void;
/**
* Adds a cell to the end of the model
*/
addCell(cellType: CellType): void;
/**
* Deletes a cell
*/
deleteCell(cellModel: ICellModel): void;
/**
* Save the model to its backing content manager.
* Serializes the model and then calls through to save it
*/
saveModel(): Promise<boolean>;
/**
* Notifies the notebook of a change in the cell
*/
onCellChange(cell: ICellModel, change: NotebookChangeType): void;
}
export interface ICellModelOptions {
notebook: INotebookModel;
isTrusted: boolean;
}
export interface ICellModel {
cellUri: URI;
id: string;
language: string;
source: string;
cellType: CellType;
trustedMode: boolean;
active: boolean;
equals(cellModel: ICellModel): boolean;
toJSON(): nb.ICell;
}
export interface IModelFactory {
createCell(cell: nb.ICell, options: ICellModelOptions): ICellModel;
createClientSession(options: IClientSessionOptions): IClientSession;
}
export interface INotebookModelOptions {
/**
* Path to the local or remote notebook
*/
path: string;
/**
* Factory for creating cells and client sessions
*/
factory: IModelFactory;
notebookManager: INotebookManager;
notificationService: INotificationService;
connectionService: IConnectionManagementService;
}
// TODO would like to move most of these constants to an extension
export namespace notebookConstants {
export const hadoopKnoxProviderName = 'HADOOP_KNOX';
export const python3 = 'python3';
export const python3DisplayName = 'Python 3';
export const defaultSparkKernel = 'pyspark3kernel';
}

94
src/sql/parts/notebook/models/notebookConnection.ts Normal file
View File

@@ -0,0 +1,94 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { localize } from 'vs/nls';
import { IConnectionProfile } from 'sql/parts/connection/common/interfaces';
export namespace constants {
export const hostPropName = 'host';
export const userPropName = 'user';
export const knoxPortPropName = 'knoxport';
export const clusterPropName = 'clustername';
export const passwordPropName = 'password';
export const defaultKnoxPort = '30443';
}
/**
* This is a temporary connection definition, with known properties for Knox gateway connections.
* Long term this should be refactored to an extension contribution
*
* @export
* @class NotebookConnection
*/
export class NotebookConnection {
private _host: string;
private _knoxPort: string;
constructor(private _connectionProfile: IConnectionProfile) {
if (!this._connectionProfile) {
throw new Error(localize('connectionInfoMissing', 'connectionInfo is required'));
}
}
public get connectionProfile(): IConnectionProfile {
return this.connectionProfile;
}
public get host(): string {
if (!this._host) {
this.ensureHostAndPort();
}
return this._host;
}
/**
* Sets host and port values, using any ',' or ':' delimited port in the hostname in
* preference to the built in port.
*/
private ensureHostAndPort(): void {
this._host = this.connectionProfile.options[constants.hostPropName];
this._knoxPort = NotebookConnection.getKnoxPortOrDefault(this.connectionProfile);
// determine whether the host has either a ',' or ':' in it
this.setHostAndPort(',');
this.setHostAndPort(':');
}
// set port and host correctly after we've identified that a delimiter exists in the host name
private setHostAndPort(delimeter: string): void {
let originalHost = this._host;
let index = originalHost.indexOf(delimeter);
if (index > -1) {
this._host = originalHost.slice(0, index);
this._knoxPort = originalHost.slice(index + 1);
}
}
public get user(): string {
return this.connectionProfile.options[constants.userPropName];
}
public get password(): string {
return this.connectionProfile.options[constants.passwordPropName];
}
public get knoxport(): string {
if (!this._knoxPort) {
this.ensureHostAndPort();
}
return this._knoxPort;
}
private static getKnoxPortOrDefault(connectionProfile: IConnectionProfile): string {
let port = connectionProfile.options[constants.knoxPortPropName];
if (!port) {
port = constants.defaultKnoxPort;
}
return port;
}
}

474
src/sql/parts/notebook/models/notebookModel.ts Normal file
View File

@@ -0,0 +1,474 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { nb } from 'sqlops';
import { localize } from 'vs/nls';
import { Event, Emitter } from 'vs/base/common/event';
import { Disposable } from 'vs/base/common/lifecycle';
import { CellModel } from './cell';
import { IClientSession, INotebookModel, IDefaultConnection, INotebookModelOptions, ICellModel, notebookConstants } from './modelInterfaces';
import { NotebookChangeType, CellTypes, CellType } from 'sql/parts/notebook/models/contracts';
import { nbversion } from '../notebookConstants';
import * as notebookUtils from '../notebookUtils';
import { INotebookManager } from 'sql/services/notebook/notebookService';
import { SparkMagicContexts } from 'sql/parts/notebook/models/sparkMagicContexts';
import { IConnectionProfile } from 'sql/parts/connection/common/interfaces';
import { NotebookConnection } from 'sql/parts/notebook/models/notebookConnection';
/*
* Used to control whether a message in a dialog/wizard is displayed as an error,
* warning, or informational message. Default is error.
*/
export enum MessageLevel {
Error = 0,
Warning = 1,
Information = 2
}
export class ErrorInfo {
constructor(public readonly message: string, public readonly severity: MessageLevel) {
}
}
export interface NotebookContentChange {
/**
* What was the change that occurred?
*/
changeType: NotebookChangeType;
/**
* Optional cells that were changed
*/
cells?: ICellModel | ICellModel[];
/**
* Optional index of the change, indicating the cell at which an insert or
* delete occurred
*/
cellIndex?: number;
/**
* Optional value indicating if the notebook is in a dirty or clean state after this change
*
* @type {boolean}
* @memberof NotebookContentChange
*/
isDirty?: boolean;
}
export class NotebookModel extends Disposable implements INotebookModel {
private _contextsChangedEmitter = new Emitter<void>();
private _contentChangedEmitter = new Emitter<NotebookContentChange>();
private _kernelsChangedEmitter = new Emitter<nb.IKernelSpec>();
private _inErrorState: boolean = false;
private _clientSession: IClientSession;
private _sessionLoadFinished: Promise<void>;
private _onClientSessionReady = new Emitter<IClientSession>();
private _activeContexts: IDefaultConnection;
private _trustedMode: boolean;
private _cells: ICellModel[];
private _defaultLanguageInfo: nb.ILanguageInfo;
private onErrorEmitter = new Emitter<ErrorInfo>();
private _savedKernelInfo: nb.IKernelInfo;
private readonly _nbformat: number = nbversion.MAJOR_VERSION;
private readonly _nbformatMinor: number = nbversion.MINOR_VERSION;
private _hadoopConnection: NotebookConnection;
private _defaultKernel: nb.IKernelSpec;
constructor(private notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) {
super();
if (!notebookOptions || !notebookOptions.path || !notebookOptions.notebookManager) {
throw new Error('path or notebook service not defined');
}
if (startSessionImmediately) {
this.backgroundStartSession();
}
this._trustedMode = false;
}
public get notebookManager(): INotebookManager {
return this.notebookOptions.notebookManager;
}
public get hasServerManager(): boolean {
// If the service has a server manager, then we can show the start button
return !!this.notebookManager.serverManager;
}
public get contentChanged(): Event<NotebookContentChange> {
return this._contentChangedEmitter.event;
}
public get isSessionReady(): boolean {
return !!this._clientSession;
}
/**
* ClientSession object which handles management of a session instance,
* plus startup of the session manager which can return key metadata about the
* notebook environment
*/
public get clientSession(): IClientSession {
return this._clientSession;
}
public get kernelChanged(): Event<nb.IKernelChangedArgs> {
return this.clientSession.kernelChanged;
}
public get kernelsChanged(): Event<nb.IKernelSpec> {
return this._kernelsChangedEmitter.event;
}
public get defaultKernel(): nb.IKernelSpec {
return this._defaultKernel;
}
public get contextsChanged(): Event<void> {
return this._contextsChangedEmitter.event;
}
public get cells(): ICellModel[] {
return this._cells;
}
public get contexts(): IDefaultConnection {
return this._activeContexts;
}
public get specs(): nb.IAllKernels | undefined {
return this.notebookManager.sessionManager.specs;
}
public get inErrorState(): boolean {
return this._inErrorState;
}
public get onError(): Event<ErrorInfo> {
return this.onErrorEmitter.event;
}
public get trustedMode(): boolean {
return this._trustedMode;
}
public set trustedMode(isTrusted: boolean) {
this._trustedMode = isTrusted;
if (this._cells) {
this._cells.forEach(c => {
c.trustedMode = this._trustedMode;
});
}
}
/**
* Indicates the server has finished loading. It may have failed to load in
* which case the view will be in an error state.
*/
public get sessionLoadFinished(): Promise<void> {
return this._sessionLoadFinished;
}
/**
* Notifies when the client session is ready for use
*/
public get onClientSessionReady(): Event<IClientSession> {
return this._onClientSessionReady.event;
}
public async requestModelLoad(isTrusted: boolean = false): Promise<void> {
try {
this._trustedMode = isTrusted;
let contents = await this.notebookManager.contentManager.getNotebookContents(this.notebookOptions.path);
let factory = this.notebookOptions.factory;
// if cells already exist, create them with language info (if it is saved)
this._cells = undefined;
if (contents) {
this._defaultLanguageInfo = this.getDefaultLanguageInfo(contents);
this._savedKernelInfo = this.getSavedKernelInfo(contents);
if (contents.cells && contents.cells.length > 0) {
this._cells = contents.cells.map(c => factory.createCell(c, { notebook: this, isTrusted: isTrusted }));
}
}
if (!this._cells) {
this._cells = [this.createCell(CellTypes.Code)];
}
} catch (error) {
this._inErrorState = true;
throw error;
}
}
addCell(cellType: CellType): void {
if (this.inErrorState || !this._cells) {
return;
}
let cell = this.createCell(cellType);
this._cells.push(cell);
this._contentChangedEmitter.fire({
changeType: NotebookChangeType.CellsAdded,
cells: [cell]
});
}
private createCell(cellType: CellType): ICellModel {
let singleCell: nb.ICell = {
cell_type: cellType,
source: '',
metadata: {},
execution_count: 1
};
return this.notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true });
}
deleteCell(cellModel: CellModel): void {
if (this.inErrorState || !this._cells) {
return;
}
let index = this._cells.findIndex((cell) => cell.equals(cellModel));
if (index > -1) {
this._cells.splice(index, 1);
this._contentChangedEmitter.fire({
changeType: NotebookChangeType.CellDeleted,
cells: [cellModel],
cellIndex: index
});
} else {
this.notifyError(localize('deleteCellFailed', 'Failed to delete cell.'));
}
}
private notifyError(error: string): void {
this.onErrorEmitter.fire(new ErrorInfo(error, MessageLevel.Error));
}
public backgroundStartSession(): void {
this._clientSession = this.notebookOptions.factory.createClientSession({
path: this.notebookOptions.path,
notebookManager: this.notebookManager,
notificationService: this.notebookOptions.notificationService
});
let id: string = this.connectionProfile ? this.connectionProfile.id : undefined;
this._hadoopConnection = this.connectionProfile ? new NotebookConnection(this.connectionProfile) : undefined;
this._clientSession.initialize(this._hadoopConnection);
this._sessionLoadFinished = this._clientSession.ready.then(async () => {
if (this._clientSession.isInErrorState) {
this.setErrorState(this._clientSession.errorMessage);
} else {
this._onClientSessionReady.fire(this._clientSession);
// Once session is loaded, can use the session manager to retrieve useful info
this.loadKernelInfo();
await this.loadActiveContexts(undefined);
}
});
}
public get languageInfo(): nb.ILanguageInfo {
return this._defaultLanguageInfo;
}
private updateLanguageInfo(info: nb.ILanguageInfo) {
if (info) {
this._defaultLanguageInfo = info;
}
}
public changeKernel(displayName: string): void {
let spec = this.getSpecNameFromDisplayName(displayName);
this.doChangeKernel(spec);
}
private doChangeKernel(kernelSpec: nb.IKernelSpec): void {
this._clientSession.changeKernel(kernelSpec)
.then((kernel) => {
kernel.ready.then(() => {
if (kernel.info) {
this.updateLanguageInfo(kernel.info.language_info);
}
}, err => undefined);
return this.updateKernelInfo(kernel);
}).catch((err) => {
this.notifyError(localize('changeKernelFailed', 'Failed to change kernel: {0}', notebookUtils.getErrorMessage(err)));
// TODO should revert kernels dropdown
});
}
public changeContext(host: string): void {
try {
let newConnection: IConnectionProfile = this._activeContexts.otherConnections.find((connection) => connection.options['host'] === host);
if (!newConnection && this._activeContexts.defaultConnection.options['host'] === host) {
newConnection = this._activeContexts.defaultConnection;
}
if (newConnection) {
SparkMagicContexts.configureContext(newConnection, this.notebookOptions);
this._hadoopConnection = new NotebookConnection(newConnection);
this._clientSession.updateConnection(this._hadoopConnection);
}
} catch (err) {
let msg = notebookUtils.getErrorMessage(err);
this.notifyError(localize('changeContextFailed', 'Changing context failed: {0}', msg));
}
}
private loadKernelInfo(): void {
this.clientSession.kernelChanged(async (e) => {
await this.loadActiveContexts(e);
});
try {
let sessionManager = this.notebookManager.sessionManager;
if (sessionManager) {
let defaultKernel = SparkMagicContexts.getDefaultKernel(sessionManager.specs, this.connectionProfile, this._savedKernelInfo, this.notebookOptions.notificationService);
this._defaultKernel = defaultKernel;
this._clientSession.statusChanged(async (session) => {
if (session && session.defaultKernelLoaded === true) {
this._kernelsChangedEmitter.fire(defaultKernel);
} else if (session && !session.defaultKernelLoaded) {
this._kernelsChangedEmitter.fire({ name: notebookConstants.python3, display_name: notebookConstants.python3DisplayName });
}
});
this.doChangeKernel(defaultKernel);
}
} catch (err) {
let msg = notebookUtils.getErrorMessage(err);
this.notifyError(localize('loadKernelFailed', 'Loading kernel info failed: {0}', msg));
}
}
// Get default language if saved in notebook file
// Otherwise, default to python
private getDefaultLanguageInfo(notebook: nb.INotebook): nb.ILanguageInfo {
return notebook!.metadata!.language_info || {
name: 'python',
version: '',
mimetype: 'x-python'
};
}
// Get default kernel info if saved in notebook file
private getSavedKernelInfo(notebook: nb.INotebook): nb.IKernelInfo {
return notebook!.metadata!.kernelspec;
}
private getSpecNameFromDisplayName(displayName: string): nb.IKernelSpec {
displayName = this.sanitizeDisplayName(displayName);
let kernel: nb.IKernelSpec = this.specs.kernels.find(k => k.display_name.toLowerCase() === displayName.toLowerCase());
if (!kernel) {
return undefined; // undefined is handled gracefully in the session to default to the default kernel
} else if (!kernel.name) {
kernel.name = this.specs.defaultKernel;
}
return kernel;
}
private setErrorState(errMsg: string): void {
this._inErrorState = true;
let msg = localize('startSessionFailed', 'Could not start session: {0}', errMsg);
this.notifyError(msg);
}
public dispose(): void {
super.dispose();
this.handleClosed();
}
public async handleClosed(): Promise<void> {
try {
if (this._clientSession) {
await this._clientSession.shutdown();
this._clientSession = undefined;
}
} catch (err) {
this.notifyError(localize('shutdownError', 'An error occurred when closing the notebook: {0}', err));
}
}
private async loadActiveContexts(kernelChangedArgs: nb.IKernelChangedArgs): Promise<void> {
this._activeContexts = await SparkMagicContexts.getContextsForKernel(this.notebookOptions.connectionService, kernelChangedArgs, this.connectionProfile);
this._contextsChangedEmitter.fire();
let defaultHadoopConnection = new NotebookConnection(this.contexts.defaultConnection);
this.changeContext(defaultHadoopConnection.host);
}
/**
* Sanitizes display name to remove IP address in order to fairly compare kernels
* In some notebooks, display name is in the format <kernel> (<ip address>)
* example: PySpark (25.23.32.4)
* @param displayName Display Name for the kernel
*/
public sanitizeDisplayName(displayName: string): string {
let name = displayName;
if (name) {
let index = name.indexOf('(');
name = (index > -1) ? name.substr(0, index - 1).trim() : name;
}
return name;
}
public async saveModel(): Promise<boolean> {
let notebook = this.toJSON();
if (!notebook) {
return false;
}
await this.notebookManager.contentManager.save(this.notebookOptions.path, notebook);
this._contentChangedEmitter.fire({
changeType: NotebookChangeType.DirtyStateChanged,
isDirty: false
});
return true;
}
private async updateKernelInfo(kernel: nb.IKernel): Promise<void> {
if (kernel) {
try {
let spec = await kernel.getSpec();
this._savedKernelInfo = {
name: kernel.name,
display_name: spec.display_name,
language: spec.language
};
} catch (err) {
// Don't worry about this for now. Just use saved values
}
}
}
/**
* Serialize the model to JSON.
*/
toJSON(): nb.INotebook {
let cells: nb.ICell[] = this.cells.map(c => c.toJSON());
let metadata = Object.create(null) as nb.INotebookMetadata;
// TODO update language and kernel when these change
metadata.kernelspec = this._savedKernelInfo;
metadata.language_info = this.languageInfo;
return {
metadata,
nbformat_minor: this._nbformatMinor,
nbformat: this._nbformat,
cells
};
}
onCellChange(cell: CellModel, change: NotebookChangeType): void {
let changeInfo: NotebookContentChange = {
changeType: change,
cells: [cell]
};
switch (change) {
case NotebookChangeType.CellOutputUpdated:
case NotebookChangeType.CellSourceUpdated:
changeInfo.changeType = NotebookChangeType.DirtyStateChanged;
changeInfo.isDirty = true;
break;
default:
// Do nothing for now
}
this._contentChangedEmitter.fire(changeInfo);
}
}

194
src/sql/parts/notebook/models/sparkMagicContexts.ts Normal file
View File

@@ -0,0 +1,194 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as path from 'path';
import { nb } from 'sqlops';
import * as json from 'vs/base/common/json';
import * as pfs from 'vs/base/node/pfs';
import { localize } from 'vs/nls';
import { IConnectionProfile } from 'sql/parts/connection/common/interfaces';
import { IDefaultConnection, notebookConstants, INotebookModelOptions } from 'sql/parts/notebook/models/modelInterfaces';
import * as notebookUtils from '../notebookUtils';
import { INotificationService } from 'vs/platform/notification/common/notification';
import { IConnectionManagementService } from 'sql/parts/connection/common/connectionManagement';
export class SparkMagicContexts {
public static get DefaultContext(): IDefaultConnection {
// TODO NOTEBOOK REFACTOR fix default connection handling
let defaultConnection: IConnectionProfile = <any> {
providerName: notebookConstants.hadoopKnoxProviderName,
id: '-1',
options:
{
host: localize('selectConnection', 'Select Connection')
}
};
return {
// default context if no other contexts are applicable
defaultConnection: defaultConnection,
otherConnections: [defaultConnection]
};
}
/**
* Get all of the applicable contexts for a given kernel
* @param apiWrapper ApiWrapper
* @param kernelChangedArgs kernel changed args (both old and new kernel info)
* @param profile current connection profile
*/
public static async getContextsForKernel(connectionService: IConnectionManagementService, kernelChangedArgs?: nb.IKernelChangedArgs, profile?: IConnectionProfile): Promise<IDefaultConnection> {
let connections: IDefaultConnection = this.DefaultContext;
if (!profile) {
if (!kernelChangedArgs || !kernelChangedArgs.newValue ||
(kernelChangedArgs.oldValue && kernelChangedArgs.newValue.id === kernelChangedArgs.oldValue.id)) {
// nothing to do, kernels are the same or new kernel is undefined
return connections;
}
}
if (kernelChangedArgs && kernelChangedArgs.newValue && kernelChangedArgs.newValue.name) {
switch (kernelChangedArgs.newValue.name) {
case (notebookConstants.python3):
// python3 case, use this.DefaultContext for the only connection
break;
//TO DO: Handle server connections based on kernel type. Right now, we call the same method for all kernel types.
default:
connections = await this.getActiveContexts(connectionService, profile);
}
} else {
connections = await this.getActiveContexts(connectionService, profile);
}
return connections;
}
/**
* Get all active contexts and sort them
* @param apiWrapper ApiWrapper
* @param profile current connection profile
*/
public static async getActiveContexts(connectionService: IConnectionManagementService, profile: IConnectionProfile): Promise<IDefaultConnection> {
let defaultConnection: IConnectionProfile = SparkMagicContexts.DefaultContext.defaultConnection;
let activeConnections: IConnectionProfile[] = await connectionService.getActiveConnections();
// If no connections exist, only show 'n/a'
if (activeConnections.length === 0) {
return SparkMagicContexts.DefaultContext;
}
activeConnections = activeConnections.filter(conn => conn.providerName === notebookConstants.hadoopKnoxProviderName);
// If launched from the right click or server dashboard, connection profile data exists, so use that as default
if (profile && profile.options) {
let profileConnection = activeConnections.filter(conn => conn.options['host'] === profile.options['host']);
if (profileConnection) {
defaultConnection = profileConnection[0];
}
} else {
if (activeConnections.length > 0) {
defaultConnection = activeConnections[0];
} else {
// TODO NOTEBOOK REFACTOR change this so it's no longer incompatible with IConnectionProfile
defaultConnection = <IConnectionProfile> <any>{
providerName: notebookConstants.hadoopKnoxProviderName,
id: '-1',
options:
{
host: localize('addConnection', 'Add new connection')
}
};
activeConnections.push(defaultConnection);
}
}
return {
otherConnections: activeConnections,
defaultConnection: defaultConnection
};
}
public static async configureContext(connection: IConnectionProfile, options: INotebookModelOptions): Promise<object> {
let sparkmagicConfDir = path.join(notebookUtils.getUserHome(), '.sparkmagic');
// TODO NOTEBOOK REFACTOR re-enable this or move to extension. Requires config files to be available in order to work
// await notebookUtils.mkDir(sparkmagicConfDir);
// let hadoopConnection = new Connection({ options: connection.options }, undefined, connection.connectionId);
// await hadoopConnection.getCredential();
// // Default to localhost in config file.
// let creds: ICredentials = {
// 'url': 'http://localhost:8088'
// };
// let configPath = notebookUtils.getTemplatePath(options.extensionContext.extensionPath, path.join('jupyter_config', 'sparkmagic_config.json'));
// let fileBuffer: Buffer = await pfs.readFile(configPath);
// let fileContents: string = fileBuffer.toString();
// let config: ISparkMagicConfig = json.parse(fileContents);
// SparkMagicContexts.updateConfig(config, creds, sparkmagicConfDir);
// let configFilePath = path.join(sparkmagicConfDir, 'config.json');
// await pfs.writeFile(configFilePath, JSON.stringify(config));
return {'SPARKMAGIC_CONF_DIR': sparkmagicConfDir};
}
/**
*
* @param specs kernel specs (comes from session manager)
* @param connectionInfo connection profile
* @param savedKernelInfo kernel info loaded from
*/
public static getDefaultKernel(specs: nb.IAllKernels, connectionInfo: IConnectionProfile, savedKernelInfo: nb.IKernelInfo, notificationService: INotificationService): nb.IKernelSpec {
let defaultKernel = specs.kernels.find((kernel) => kernel.name === specs.defaultKernel);
let profile = connectionInfo as IConnectionProfile;
if (specs && connectionInfo && profile.providerName === notebookConstants.hadoopKnoxProviderName) {
// set default kernel to default spark kernel if profile exists
// otherwise, set default to kernel info loaded from existing file
defaultKernel = !savedKernelInfo ? specs.kernels.find((spec) => spec.name === notebookConstants.defaultSparkKernel) : savedKernelInfo;
} else {
// Handle kernels
if (savedKernelInfo && savedKernelInfo.name.toLowerCase().indexOf('spark') > -1) {
notificationService.warn(localize('sparkKernelRequiresConnection', 'Cannot use kernel {0} as no connection is active. The default kernel of Python3 will be used instead.', savedKernelInfo.display_name));
}
}
// If no default kernel specified (should never happen), default to python3
if (!defaultKernel) {
defaultKernel = {
name: notebookConstants.python3,
display_name: notebookConstants.python3DisplayName
};
}
return defaultKernel;
}
private static updateConfig(config: ISparkMagicConfig, creds: ICredentials, homePath: string): void {
config.kernel_python_credentials = creds;
config.kernel_scala_credentials = creds;
config.kernel_r_credentials = creds;
config.logging_config.handlers.magicsHandler.home_path = homePath;
}
}
interface ICredentials {
'url': string;
}
interface ISparkMagicConfig {
kernel_python_credentials: ICredentials;
kernel_scala_credentials: ICredentials;
kernel_r_credentials: ICredentials;
logging_config: {
handlers: {
magicsHandler: {
home_path: string;
}
}
};
}
export interface IKernelJupyterID {
id: string;
jupyterId: string;
}