Notebook: Re-Enable Attach to Dropdown Functionality (#3250)

* first attach to working

* Transfer changes from sqlopsstudioextensions PR 448

* Transfer changes from sqlopsstudioextensions PR 447

* Transfer changes from sqlopsstudioextensions PR 456

* Transfer changes from sqlopsstudioextensions PR 465

* Transfer changes from sqlopsstudioextensions PR 463

* Transfer changes from sqlopsstudioextensions PR 482

* Transfer changes from sqlopsstudioextensions PR 485

* Session and Kernel implementation except executeRequest

* Attach to port compiling

* Further tweaks to attach to dropdown, re-enable opening connection dialog

* Revert "Merge remote-tracking branch 'origin/Notebook/sessionExtension' into feature/workingAttachTo"

This reverts commit 94703db87c85416c4ae36762afc1094d6e71166a, reversing
changes made to e4dc25331036d259e9c762cfe8741f957bb5c590.

* Fix code formatting

* Fix for new Add new connection issue

src/sql/parts/notebook/models/cell.ts

@@ -156,7 +156,7 @@ export class CellModel implements ICellModel {
 		future.setIOPubHandler({ handle: (msg) => this.handleIOPub(msg) });
 	}
 
-	private clearOutputs(): void {
+	public clearOutputs(): void {
 		this._outputs = [];
 		this.fireOutputsChanged();
 	}
@@ -172,7 +172,7 @@ export class CellModel implements ICellModel {
 		}
 	}
 
-	public get outputs(): ReadonlyArray<nb.ICellOutput> {
+	public get outputs(): Array<nb.ICellOutput> {
 		return this._outputs;
 	}
 
@@ -220,6 +220,8 @@ export class CellModel implements ICellModel {
 		//     this._displayIdMap.set(displayId, targets);
 		// }
 		if (output) {
+			// deletes transient node in the serialized JSON
+			delete output['transient'];
 			this._outputs.push(output);
 			this.fireOutputsChanged();
 		}
@@ -241,7 +243,6 @@ export class CellModel implements ICellModel {
 			cellJson.metadata.language = this._language,
 			cellJson.outputs = this._outputs;
 			cellJson.execution_count = 1; // TODO: keep track of actual execution count
-
 		}
 		return cellJson as nb.ICell;
 	}

src/sql/parts/notebook/models/clientSession.ts

@@ -276,7 +276,7 @@ export class ClientSession implements IClientSession {
 
 	public async updateConnection(connection: NotebookConnection): Promise<void> {
 		if (!this.kernel) {
-			// TODO is there any case where skipping causes errors? Do far it seems like it gets called twice
+			// TODO is there any case where skipping causes errors? So far it seems like it gets called twice
 			return;
 		}
 		this._connection = (connection.connectionProfile.id !== '-1') ? connection : this._connection;

src/sql/parts/notebook/models/modelInterfaces.ts

@@ -288,7 +288,7 @@ export interface INotebookModel {
 	readonly contexts: IDefaultConnection | undefined;
 
 	/**
-	 * The trusted mode of the NoteBook
+	 * The trusted mode of the Notebook
 	 */
 	trustedMode: boolean;
 
@@ -301,7 +301,12 @@ export interface INotebookModel {
 	/**
 	 * Change the current context (if applicable)
 	 */
-	changeContext(host: string): void;
+	changeContext(host: string, connection?: IConnectionProfile): void;
+
+	/**
+	 * Find a cell's index given its model
+	 */
+	findCellIndex(cellModel: ICellModel): number;
 
 	/**
 	 * Adds a cell to the index of the model
@@ -380,5 +385,5 @@ export namespace notebookConstants {
 	export const python3 = 'python3';
 	export const python3DisplayName = 'Python 3';
 	export const defaultSparkKernel = 'pyspark3kernel';
-
+	export const hostPropName = 'host';
 }

src/sql/parts/notebook/models/notebookModel.ts

@@ -5,7 +5,7 @@
 
 'use strict';
 
-import { nb } from 'sqlops';
+import { nb, connection } from 'sqlops';
 
 import { localize } from 'vs/nls';
 import { Event, Emitter } from 'vs/base/common/event';
@@ -167,6 +167,10 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		}
 	}
 
+	public get hadoopConnection(): NotebookConnection {
+		return this._hadoopConnection;
+	}
+
 	/**
 	 * Indicates the server has finished loading. It may have failed to load in
 	 * which case the view will be in an error state.
@@ -186,7 +190,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		try {
 			this._trustedMode = isTrusted;
 			let contents = null;
-			if(this.notebookOptions.notebookUri.scheme !== Schemas.untitled) {
+			if (this.notebookOptions.notebookUri.scheme !== Schemas.untitled) {
 				contents = await this.notebookManager.contentManager.getNotebookContents(this.notebookOptions.notebookUri);
 			}
 			let factory = this.notebookOptions.factory;
@@ -208,25 +212,29 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		}
 	}
 
+	public findCellIndex(cellModel: CellModel): number {
+		return this._cells.findIndex((cell) => cell.equals(cellModel));
+	}
+
 	public addCell(cellType: CellType, index?: number): void {
-        if (this.inErrorState || !this._cells) {
-            return;
-        }
-        let cell = this.createCell(cellType);
+		if (this.inErrorState || !this._cells) {
+			return;
+		}
+		let cell = this.createCell(cellType);
 
-        if (index !== undefined && index !== null && index >= 0 && index < this._cells.length) {
-            this._cells.splice(index, 0, cell);
-        } else {
-            this._cells.push(cell);
-            index = undefined;
-        }
+		if (index !== undefined && index !== null && index >= 0 && index < this._cells.length) {
+			this._cells.splice(index, 0, cell);
+		} else {
+			this._cells.push(cell);
+			index = undefined;
+		}
 
-        this._contentChangedEmitter.fire({
-            changeType: NotebookChangeType.CellsAdded,
-            cells: [cell],
-            cellIndex: index
-        });
-    }
+		this._contentChangedEmitter.fire({
+			changeType: NotebookChangeType.CellsAdded,
+			cells: [cell],
+			cellIndex: index
+		});
+	}
 
 	private createCell(cellType: CellType): ICellModel {
 		let singleCell: nb.ICell = {
@@ -283,12 +291,16 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			} else {
 				this._onClientSessionReady.fire(this._clientSession);
 				// Once session is loaded, can use the session manager to retrieve useful info
-					this.loadKernelInfo();
-					await this.loadActiveContexts(undefined);
+				this.loadKernelInfo();
+				await this.loadActiveContexts(undefined);
 			}
 		});
 	}
 
+	private isValidKnoxConnection(profile: IConnectionProfile | connection.Connection) {
+		return profile && profile.providerName === notebookConstants.hadoopKnoxProviderName && profile.options[notebookConstants.hostPropName] !== undefined;
+	}
+
 	public get languageInfo(): nb.ILanguageInfo {
 		return this._defaultLanguageInfo;
 	}
@@ -304,9 +316,9 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		this.doChangeKernel(spec);
 	}
 
-	private doChangeKernel(kernelSpec: nb.IKernelSpec): void {
-		this._clientSession.changeKernel(kernelSpec)
-		.then((kernel) => {
+	public doChangeKernel(kernelSpec: nb.IKernelSpec): Promise<void> {
+		return this._clientSession.changeKernel(kernelSpec)
+			.then((kernel) => {
 				kernel.ready.then(() => {
 					if (kernel.info) {
 						this.updateLanguageInfo(kernel.info.language_info);
@@ -317,26 +329,40 @@ export class NotebookModel extends Disposable implements INotebookModel {
 				this.notifyError(localize('changeKernelFailed', 'Failed to change kernel: {0}', notebookUtils.getErrorMessage(err)));
 				// TODO should revert kernels dropdown
 			});
-
 	}
 
-	public changeContext(host: string): void {
+	public changeContext(host: string, newConnection?: IConnectionProfile): void {
 		try {
-			let newConnection: IConnectionProfile = this._activeContexts.otherConnections.find((connection) => connection.options['host'] === host);
+			if (!newConnection) {
+				newConnection = this._activeContexts.otherConnections.find((connection) => connection.options['host'] === host);
+			}
 			if (!newConnection && this._activeContexts.defaultConnection.options['host'] === host) {
 				newConnection = this._activeContexts.defaultConnection;
 			}
-			if (newConnection) {
-				SparkMagicContexts.configureContext(newConnection, this.notebookOptions);
-				this._hadoopConnection = new NotebookConnection(newConnection);
-				this._clientSession.updateConnection(this._hadoopConnection);
-			}
+			SparkMagicContexts.configureContext(this.notebookOptions);
+			this._hadoopConnection = new NotebookConnection(newConnection);
+			this.refreshConnections(newConnection);
+			this._clientSession.updateConnection(this._hadoopConnection);
 		} catch (err) {
 			let msg = notebookUtils.getErrorMessage(err);
 			this.notifyError(localize('changeContextFailed', 'Changing context failed: {0}', msg));
 		}
 	}
 
+	private refreshConnections(newConnection: IConnectionProfile) {
+		if (this.isValidKnoxConnection(newConnection) &&
+			this._hadoopConnection.connectionProfile.id !== '-1' &&
+			this._hadoopConnection.connectionProfile.id !== this._activeContexts.defaultConnection.id) {
+			// Put the defaultConnection to the head of otherConnections
+			if (this.isValidKnoxConnection(this._activeContexts.defaultConnection)) {
+				this._activeContexts.otherConnections = this._activeContexts.otherConnections.filter(conn => conn.id !== this._activeContexts.defaultConnection.id);
+				this._activeContexts.otherConnections.unshift(this._activeContexts.defaultConnection);
+			}
+			// Change the defaultConnection to newConnection
+			this._activeContexts.defaultConnection = newConnection;
+		}
+	}
+
 	private loadKernelInfo(): void {
 		this.clientSession.kernelChanged(async (e) => {
 			await this.loadActiveContexts(e);
@@ -413,8 +439,10 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private async loadActiveContexts(kernelChangedArgs: nb.IKernelChangedArgs): Promise<void> {
 		this._activeContexts = await SparkMagicContexts.getContextsForKernel(this.notebookOptions.connectionService, kernelChangedArgs, this.connectionProfile);
 		this._contextsChangedEmitter.fire();
-		let defaultHadoopConnection = new NotebookConnection(this.contexts.defaultConnection);
-		this.changeContext(defaultHadoopConnection.host);
+		if (this.contexts.defaultConnection !== undefined && this.contexts.defaultConnection.options !== undefined) {
+			let defaultHadoopConnection = new NotebookConnection(this.contexts.defaultConnection);
+			this.changeContext(defaultHadoopConnection.host);
+		}
 	}
 
 	/**
@@ -488,7 +516,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 				changeInfo.isDirty = true;
 				break;
 			default:
-				// Do nothing for now
+			// Do nothing for now
 		}
 		this._contentChangedEmitter.fire(changeInfo);
 	}

src/sql/parts/notebook/models/sparkMagicContexts.ts

@@ -26,7 +26,7 @@ export class SparkMagicContexts {
 			id: '-1',
 			options:
 			{
-				host: localize('selectConnection', 'Select Connection')
+				host: localize('selectConnection', 'Select connection')
 			}
 		};
 
@@ -76,10 +76,13 @@ export class SparkMagicContexts {
 		let defaultConnection: IConnectionProfile = SparkMagicContexts.DefaultContext.defaultConnection;
 		let activeConnections: IConnectionProfile[] = await connectionService.getActiveConnections();
 		// If no connections exist, only show 'n/a'
-		if (activeConnections.length === 0) {
-			return SparkMagicContexts.DefaultContext;
-		}
-		activeConnections = activeConnections.filter(conn => conn.providerName === notebookConstants.hadoopKnoxProviderName);
+        if (activeConnections && activeConnections.length > 0) {
+            // Remove all non-Spark connections
+            activeConnections = activeConnections.filter(conn => conn.providerName === notebookConstants.hadoopKnoxProviderName);
+        }
+        if (activeConnections.length === 0) {
+            return SparkMagicContexts.DefaultContext;
+        }
 
 		// If launched from the right click or server dashboard, connection profile data exists, so use that as default
 		if (profile && profile.options) {
@@ -109,13 +112,11 @@ export class SparkMagicContexts {
 		};
 	}
 
-	public static async configureContext(connection: IConnectionProfile, options: INotebookModelOptions): Promise<object> {
+	public static async configureContext(options: INotebookModelOptions): Promise<object> {
 		let sparkmagicConfDir = path.join(notebookUtils.getUserHome(), '.sparkmagic');
 		// TODO NOTEBOOK REFACTOR re-enable this or move to extension. Requires config files to be available in order to work
 		// await notebookUtils.mkDir(sparkmagicConfDir);
 
-		// let hadoopConnection = new Connection({ options: connection.options }, undefined, connection.connectionId);
-		// await hadoopConnection.getCredential();
 		// // Default to localhost in config file.
 		// let creds: ICredentials = {
 		//     'url': 'http://localhost:8088'