Integrate first SQL Notebooks Bits into Master (#3679)

* First crack tsql notebook (no output rendered yet)

* getting messages back

* intellisense working first cell, no connection errors

* sql notebook cell output functioning

* Latest SQL noteobook changes

* Undo change to launch.json

* Plumbing providers through

* Kernels shown from multiple providers, can switch between them. No mementos yet

* Ensure we have a feature flag for SQL notebooks, ensure existing functionality still works

* Fix tslint duplicate imports issue

* Addressing PR comments

* second round of PR feedback to cleanup notebook service manager code

* merge latest from master

src/sql/services/notebook/notebookRegistry.ts

@@ -18,11 +18,12 @@ export const Extensions = {
 export interface NotebookProviderRegistration {
 	provider: string;
 	fileExtensions: string | string[];
+	standardKernels: string | string[];
 }
 
 let notebookProviderType: IJSONSchema = {
 	type: 'object',
-	default: { provider: '', fileExtensions: [] },
+	default: { provider: '', fileExtensions: [], standardKernels: [] },
 	properties: {
 		provider: {
 			description: localize('carbon.extension.contributes.notebook.provider', 'Identifier of the notebook provider.'),
@@ -39,6 +40,18 @@ let notebookProviderType: IJSONSchema = {
 					}
 				}
 			]
+		},
+		standardKernels: {
+			description: localize('carbon.extension.contributes.notebook.standardKernels', 'What kernels should be standard with this notebook provider'),
+			oneOf: [
+				{ type: 'string' },
+				{
+					type: 'array',
+					items: {
+						type: 'string'
+					}
+				}
+			]
 		}
 	}
 };

src/sql/services/notebook/notebookService.ts

@@ -23,6 +23,7 @@ export const INotebookService = createDecorator<INotebookService>(SERVICE_ID);
 
 export const DEFAULT_NOTEBOOK_PROVIDER = 'builtin';
 export const DEFAULT_NOTEBOOK_FILETYPE = 'IPYNB';
+export const SQL_NOTEBOOK_PROVIDER = 'sql';
 
 export interface INotebookService {
 	_serviceBrand: any;
@@ -45,7 +46,7 @@ export interface INotebookService {
 
 	getSupportedFileExtensions(): string[];
 
-	getProviderForFileType(fileType: string): string;
+	getProvidersForFileType(fileType: string): string[];
 
 	/**
 	 * Initializes and returns a Notebook manager that can handle all important calls to open, display, and
@@ -86,6 +87,7 @@ export interface INotebookParams extends IBootstrapParams {
 	notebookUri: URI;
 	input: NotebookInput;
 	providerId: string;
+	providers: string[];
 	isTrusted: boolean;
 	profile?: IConnectionProfile;
 	modelFactory?: ModelFactory;

src/sql/services/notebook/notebookServiceImpl.ts

@@ -12,7 +12,7 @@ import { Registry } from 'vs/platform/registry/common/platform';
 
 import {
 	INotebookService, INotebookManager, INotebookProvider, DEFAULT_NOTEBOOK_PROVIDER,
-	DEFAULT_NOTEBOOK_FILETYPE, INotebookEditor
+	DEFAULT_NOTEBOOK_FILETYPE, INotebookEditor, SQL_NOTEBOOK_PROVIDER
 } from 'sql/services/notebook/notebookService';
 import { RenderMimeRegistry } from 'sql/parts/notebook/outputs/registry';
 import { standardRendererFactories } from 'sql/parts/notebook/outputs/factories';
@@ -27,6 +27,9 @@ import { IExtensionManagementService, IExtensionIdentifier } from 'vs/platform/e
 import { Disposable } from 'vs/base/common/lifecycle';
 import { getIdFromLocalExtensionId } from 'vs/platform/extensionManagement/common/extensionManagementUtil';
 import { Deferred } from 'sql/base/common/promise';
+import { SqlSessionManager } from 'sql/services/notebook/sqlSessionManager';
+import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
+import { sqlNotebooksEnabled } from 'sql/parts/notebook/notebookUtils';
 
 export interface NotebookProviderProperties {
 	provider: string;
@@ -70,24 +73,25 @@ export class NotebookService extends Disposable implements INotebookService {
 	private _memento = new Memento('notebookProviders');
 	private _mimeRegistry: RenderMimeRegistry;
 	private _providers: Map<string, ProviderDescriptor> = new Map();
-	private _managers: Map<string, INotebookManager> = new Map();
+	private _managersMap: Map<string, INotebookManager[]> = new Map();
 	private _onNotebookEditorAdd = new Emitter<INotebookEditor>();
 	private _onNotebookEditorRemove = new Emitter<INotebookEditor>();
 	private _onCellChanged = new Emitter<INotebookEditor>();
 	private _onNotebookEditorRename = new Emitter<INotebookEditor>();
 	private _editors = new Map<string, INotebookEditor>();
-	private _fileToProviders = new Map<string, NotebookProviderRegistration>();
+	private _fileToProviders = new Map<string, NotebookProviderRegistration[]>();
 	private _registrationComplete = new Deferred<void>();
 	private _isRegistrationComplete = false;
 
 	constructor(
 		@IStorageService private _storageService: IStorageService,
 		@IExtensionService extensionService: IExtensionService,
-		@IExtensionManagementService extensionManagementService: IExtensionManagementService
+		@IExtensionManagementService extensionManagementService: IExtensionManagementService,
+		@IInstantiationService private _instantiationService: IInstantiationService
 	) {
 		super();
 		this._register(notebookRegistry.onNewRegistration(this.updateRegisteredProviders, this));
-		this.registerDefaultProvider();
+		this.registerBuiltInProvider();
 
 		if (extensionService) {
 				extensionService.whenInstalledExtensionsRegistered().then(() => {
@@ -142,25 +146,33 @@ export class NotebookService extends Disposable implements INotebookService {
 	}
 
 	private addFileProvider(fileType: string, provider: NotebookProviderRegistration) {
-		this._fileToProviders.set(fileType.toUpperCase(), provider);
+		let providers = this._fileToProviders.get(fileType.toUpperCase());
+		if (!providers) {
+			providers = [];
+		}
+		providers.push(provider);
+		this._fileToProviders.set(fileType.toUpperCase(), providers);
 	}
 
 	getSupportedFileExtensions(): string[] {
 		return Array.from(this._fileToProviders.keys());
 	}
 
-	getProviderForFileType(fileType: string): string {
+	getProvidersForFileType(fileType: string): string[] {
 		fileType = fileType.toUpperCase();
-		let provider = this._fileToProviders.get(fileType);
-		return provider ? provider.provider : undefined;
+		let providers = this._fileToProviders.get(fileType);
+
+		return providers ? providers.map(provider => provider.provider) : undefined;
 	}
 
 	public shutdown(): void {
-		this._managers.forEach(manager => {
-			if (manager.serverManager) {
-				// TODO should this thenable be awaited?
-				manager.serverManager.stopServer();
-			}
+		this._managersMap.forEach(manager => {
+			manager.forEach(m => {
+				if (m.serverManager) {
+					// TODO should this thenable be awaited?
+					m.serverManager.stopServer();
+				}
+			});
 		});
 	}
 
@@ -169,14 +181,20 @@ export class NotebookService extends Disposable implements INotebookService {
 			throw new Error(localize('notebookUriNotDefined', 'No URI was passed when creating a notebook manager'));
 		}
 		let uriString = uri.toString();
-		let manager = this._managers.get(uriString);
-		if (!manager) {
-			manager = await this.doWithProvider(providerId, (provider) => provider.getNotebookManager(uri));
-			if (manager) {
-				this._managers.set(uriString, manager);
+		let managers: INotebookManager[] = this._managersMap.get(uriString);
+		// If manager already exists for a given notebook, return it
+		if (managers) {
+			let index = managers.findIndex(m => m.providerId === providerId);
+			if (index && index >= 0) {
+				return managers[index];
 			}
 		}
-		return manager;
+		let newManager = await this.doWithProvider(providerId, (provider) => provider.getNotebookManager(uri));
+
+		managers = managers || [];
+		managers.push(newManager);
+		this._managersMap.set(uriString, managers);
+		return newManager;
 	}
 
 	get onNotebookEditorAdd(): Event<INotebookEditor> {
@@ -226,12 +244,14 @@ export class NotebookService extends Disposable implements INotebookService {
 	private sendNotebookCloseToProvider(editor: INotebookEditor): void {
 		let notebookUri = editor.notebookParams.notebookUri;
 		let uriString = notebookUri.toString();
-		let manager = this._managers.get(uriString);
+		let manager = this._managersMap.get(uriString);
 		if (manager) {
 			// As we have a manager, we can assume provider is ready
-			this._managers.delete(uriString);
-			let provider = this._providers.get(manager.providerId);
-			provider.instance.handleNotebookClosed(notebookUri);
+			this._managersMap.delete(uriString);
+			manager.forEach(m => {
+				let provider = this._providers.get(m.providerId);
+				provider.instance.handleNotebookClosed(notebookUri);
+			});
 		}
 	}
 
@@ -303,13 +323,24 @@ export class NotebookService extends Disposable implements INotebookService {
 		}
 	}
 
-	private registerDefaultProvider() {
-		let defaultProvider = new BuiltinProvider();
-		this.registerProvider(defaultProvider.providerId, defaultProvider);
-		notebookRegistry.registerNotebookProvider({
-			provider: defaultProvider.providerId,
-			fileExtensions: DEFAULT_NOTEBOOK_FILETYPE
-		});
+	private registerBuiltInProvider() {
+		if (!sqlNotebooksEnabled()) {
+			let defaultProvider = new BuiltinProvider();
+			this.registerProvider(defaultProvider.providerId, defaultProvider);
+			notebookRegistry.registerNotebookProvider({
+				provider: defaultProvider.providerId,
+				fileExtensions: DEFAULT_NOTEBOOK_FILETYPE,
+				standardKernels: []
+			});
+		} else {
+			let sqlProvider = new SqlNotebookProvider(this._instantiationService);
+			this.registerProvider(sqlProvider.providerId, sqlProvider);
+			notebookRegistry.registerNotebookProvider({
+				provider: sqlProvider.providerId,
+				fileExtensions: DEFAULT_NOTEBOOK_FILETYPE,
+				standardKernels: ['SQL']
+			});
+		}
 	}
 
 	private removeContributedProvidersFromCache(identifier: IExtensionIdentifier, extensionService: IExtensionService) {
@@ -330,6 +361,7 @@ export class BuiltinProvider implements INotebookProvider {
 	constructor() {
 		this.manager = new BuiltInNotebookManager();
 	}
+
 	public get providerId(): string {
 		return DEFAULT_NOTEBOOK_PROVIDER;
 	}
@@ -350,6 +382,7 @@ export class BuiltInNotebookManager implements INotebookManager {
 		this._contentManager = new LocalContentManager();
 		this._sessionManager = new SessionManager();
 	}
+
 	public get providerId(): string {
 		return DEFAULT_NOTEBOOK_PROVIDER;
 	}
@@ -367,3 +400,50 @@ export class BuiltInNotebookManager implements INotebookManager {
 	}
 
 }
+
+export class SqlNotebookProvider implements INotebookProvider {
+	private manager: SqlNotebookManager;
+
+	constructor(private _instantiationService: IInstantiationService) {
+		this.manager = new SqlNotebookManager(this._instantiationService);
+	}
+
+	public get providerId(): string {
+		return SQL_NOTEBOOK_PROVIDER;
+	}
+
+	getNotebookManager(notebookUri: URI): Thenable<INotebookManager> {
+		return Promise.resolve(this.manager);
+	}
+
+	handleNotebookClosed(notebookUri: URI): void {
+		// No-op
+	}
+}
+
+export class SqlNotebookManager implements INotebookManager {
+	private _contentManager: nb.ContentManager;
+	private _sessionManager: nb.SessionManager;
+
+	constructor(private _instantiationService: IInstantiationService) {
+		this._contentManager = new LocalContentManager();
+		this._sessionManager = new SqlSessionManager(this._instantiationService);
+	}
+
+	public get providerId(): string {
+		return SQL_NOTEBOOK_PROVIDER;
+	}
+
+	public get contentManager(): nb.ContentManager {
+		return this._contentManager;
+	}
+
+	public get serverManager(): nb.ServerManager {
+		return undefined;
+	}
+
+	public get sessionManager(): nb.SessionManager {
+		return this._sessionManager;
+	}
+
+}

src/sql/services/notebook/sessionManager.ts

@@ -1,3 +1,7 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
 'use strict';
 
 import { nb } from 'sqlops';

src/sql/services/notebook/sqlSessionManager.ts

@@ -0,0 +1,325 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { nb, QueryExecuteSubsetResult, IDbColumn, DbCellValue } from 'sqlops';
+import { localize } from 'vs/nls';
+import { FutureInternal } from 'sql/parts/notebook/models/modelInterfaces';
+import QueryRunner, { EventType } from 'sql/parts/query/execution/queryRunner';
+import { IConnectionManagementService, IErrorMessageService } from 'sql/parts/connection/common/connectionManagement';
+import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
+import Severity from 'vs/base/common/severity';
+import * as Utils from 'sql/parts/connection/common/utils';
+import { Deferred } from 'sql/base/common/promise';
+import { Disposable } from 'vs/base/common/lifecycle';
+import { mssqlProviderName } from 'sql/parts/connection/common/constants';
+
+export const sqlKernel: string = localize('sqlKernel', 'SQL');
+export const sqlKernelError: string = localize("sqlKernelError", "SQL kernel error");
+
+let sqlKernelSpec: nb.IKernelSpec = ({
+	name: sqlKernel,
+	language: 'sql',
+	display_name: sqlKernel
+});
+
+export interface SQLData {
+	columns: Array<string>;
+	rows: Array<Array<string>>;
+}
+
+export class SqlSessionManager implements nb.SessionManager {
+	constructor(private _instantiationService: IInstantiationService) {}
+
+	public get isReady(): boolean {
+		return true;
+	}
+
+	public get ready(): Thenable<void> {
+		return Promise.resolve();
+	}
+
+	public get specs(): nb.IAllKernels {
+		let allKernels: nb.IAllKernels = {
+			defaultKernel: sqlKernel,
+			kernels: [sqlKernelSpec]
+		};
+		return allKernels;
+	}
+
+	startNew(options: nb.ISessionOptions): Thenable<nb.ISession> {
+		let session = new SqlSession(options, this._instantiationService);
+		return Promise.resolve(session);
+	}
+
+	shutdown(id: string): Thenable<void> {
+		return Promise.resolve();
+	}
+}
+
+export class SqlSession implements nb.ISession {
+	private _kernel: SqlKernel;
+	private _defaultKernelLoaded = false;
+
+	public set defaultKernelLoaded(value) {
+		this._defaultKernelLoaded = value;
+	}
+
+	public get defaultKernelLoaded(): boolean {
+		return this._defaultKernelLoaded;
+	}
+
+	constructor(private options: nb.ISessionOptions, private _instantiationService: IInstantiationService) {
+		this._kernel = this._instantiationService.createInstance(SqlKernel);
+	}
+
+	public get canChangeKernels(): boolean {
+		return true;
+	}
+
+	public get id(): string {
+		return this.options.kernelId || '';
+	}
+
+	public get path(): string {
+		return this.options.path;
+	}
+
+	public get name(): string {
+		return this.options.name || '';
+	}
+
+	public get type(): string {
+		return this.options.type || '';
+	}
+
+	public get status(): nb.KernelStatus {
+		return 'connected';
+	}
+
+	public get kernel(): nb.IKernel {
+		return this._kernel;
+	}
+
+	changeKernel(kernelInfo: nb.IKernelSpec): Thenable<nb.IKernel> {
+		return Promise.resolve(this.kernel);
+	}
+}
+
+class SqlKernel extends Disposable implements nb.IKernel {
+	private _queryRunner: QueryRunner;
+	private _columns: IDbColumn[];
+	private _rows: DbCellValue[][];
+
+	constructor(@IConnectionManagementService private _connectionManagementService: IConnectionManagementService,
+				@IInstantiationService private _instantiationService: IInstantiationService,
+				@IErrorMessageService private _errorMessageService: IErrorMessageService) {
+		super();
+	}
+
+	public get id(): string {
+		return '-1';
+	}
+
+	public get name(): string {
+		return sqlKernel;
+	}
+
+	public get supportsIntellisense(): boolean {
+		return true;
+	}
+
+	public get isReady(): boolean {
+		// should we be checking on the tools service status here?
+		return true;
+	}
+
+	public get ready(): Thenable<void> {
+		return Promise.resolve();
+	}
+
+	public get info(): nb.IInfoReply {
+		let info: nb.IInfoReply = {
+			protocol_version: '',
+			implementation: '',
+			implementation_version: '',
+			language_info: {
+				name: 'sql',
+				version: '',
+			},
+			banner: '',
+			help_links: [{
+				text: '',
+				url: ''
+			}]
+		};
+
+		return info;
+	}
+	getSpec(): Thenable<nb.IKernelSpec> {
+		return Promise.resolve(sqlKernelSpec);
+	}
+
+	requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture {
+		if (this._queryRunner) {
+			this._queryRunner.runQuery(content.code);
+		} else {
+			let connections = this._connectionManagementService.getActiveConnections();
+			let connectionProfile = connections.find(connection => connection.providerName === mssqlProviderName);
+			let connectionUri = Utils.generateUri(connectionProfile, 'notebook');
+			this._queryRunner = this._instantiationService.createInstance(QueryRunner, connectionUri, undefined);
+			this._connectionManagementService.connect(connectionProfile, connectionUri).then((result) =>
+			{
+				this.addQueryEventListeners(this._queryRunner);
+				this._queryRunner.runQuery(content.code);
+			});
+		}
+
+		return new SQLFuture(this._queryRunner);
+	}
+
+	requestComplete(content: nb.ICompleteRequest): Thenable<nb.ICompleteReplyMsg> {
+		let response: Partial<nb.ICompleteReplyMsg> = { };
+		return Promise.resolve(response as nb.ICompleteReplyMsg);
+	}
+
+	interrupt(): Thenable<void> {
+		return Promise.resolve(undefined);
+	}
+
+	private addQueryEventListeners(queryRunner: QueryRunner): void {
+		this._register(queryRunner.addListener(EventType.COMPLETE, () => {
+			this.queryComplete().catch(error => {
+				this._errorMessageService.showDialog(Severity.Error, sqlKernelError, error);
+			});
+		}));
+		this._register(queryRunner.addListener(EventType.MESSAGE, message => {
+			if (message.isError) {
+				this._errorMessageService.showDialog(Severity.Error, sqlKernelError, message.message);
+			}
+		}));
+	}
+
+	private async queryComplete(): Promise<void> {
+		let batches = this._queryRunner.batchSets;
+		// currently only support 1 batch set 1 resultset
+		if (batches.length > 0) {
+			let batch = batches[0];
+			if (batch.resultSetSummaries.length > 0
+				&& batch.resultSetSummaries[0].rowCount > 0
+			) {
+				let resultset = batch.resultSetSummaries[0];
+				this._columns = resultset.columnInfo;
+				let rows: QueryExecuteSubsetResult;
+				try {
+					rows = await this._queryRunner.getQueryRows(0, resultset.rowCount, batch.id, resultset.id);
+				} catch (e) {
+					return Promise.reject(e);
+				}
+				this._rows = rows.resultSubset.rows;
+			}
+		}
+		// TODO issue #2746 should ideally show a warning inside the dialog if have no data
+	}
+}
+
+export class SQLFuture extends Disposable implements FutureInternal {
+	private _msg: nb.IMessage = undefined;
+
+	constructor(private _queryRunner: QueryRunner) {
+		super();
+	}
+	get inProgress(): boolean {
+		return !this._queryRunner.hasCompleted;
+	}
+
+	get msg(): nb.IMessage {
+		return this._msg;
+	}
+
+	get done(): Thenable<nb.IShellMessage> {
+		let deferred = new Deferred<nb.IShellMessage> ();
+		try {
+			this._register(this._queryRunner.onBatchEnd(e => {
+				let msg: nb.IShellMessage = {
+					channel: 'shell',
+					type: 'execute_reply',
+					content: { status: 'ok' },
+					header: undefined,
+					metadata: {},
+					parent_header: undefined
+				};
+				this._msg = msg;
+				deferred.resolve(msg);
+			}));
+		} catch {
+			return Promise.resolve(undefined);
+		}
+		return deferred.promise;
+	}
+
+	sendInputReply(content: nb.IInputReply): void {
+		// no-op
+	}
+
+	setReplyHandler(handler: nb.MessageHandler<nb.IShellMessage>): void {
+		// no-op
+	}
+	setStdInHandler(handler: nb.MessageHandler<nb.IStdinMessage>): void {
+		// no-op
+	}
+	setIOPubHandler(handler: nb.MessageHandler<nb.IIOPubMessage>): void {
+		this._register(this._queryRunner.onBatchEnd(batch => {
+			this._queryRunner.getQueryRows(0, batch.resultSetSummaries[0].rowCount, 0, 0).then(d => {
+				let data:SQLData = {
+					columns: batch.resultSetSummaries[0].columnInfo.map(c => c.columnName),
+					rows: d.resultSubset.rows.map(r => r.map(c => c.displayValue))
+				};
+				let table: HTMLTableElement = document.createElement('table');
+				table.createTHead();
+				table.createTBody();
+				let hrow = <HTMLTableRowElement>table.insertRow();
+				// headers
+				for (let column of data.columns) {
+					var cell = hrow.insertCell();
+					cell.innerHTML = column;
+				}
+
+				for (let row in data.rows) {
+					let hrow = <HTMLTableRowElement>table.insertRow();
+					for (let column in data.columns) {
+						var cell = hrow.insertCell();
+						cell.innerHTML = data.rows[row][column];
+					}
+				}
+				let tableHtml = '<table>' + table.innerHTML + '</table>';
+
+				let msg: nb.IIOPubMessage = {
+					channel: 'iopub',
+					type: 'iopub',
+					header: <nb.IHeader> {
+						msg_id: undefined,
+						msg_type: 'execute_result'
+					},
+					content: <nb.IExecuteResult> {
+						output_type: 'execute_result',
+						metadata: {},
+						execution_count: 0,
+						data: { 'text/html' : tableHtml},
+					},
+					metadata: undefined,
+					parent_header: undefined
+				};
+				handler.handle(msg);
+			});
+		}));
+	}
+	registerMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | Thenable<boolean>): void {
+		// no-op
+	}
+	removeMessageHook(hook: (msg: nb.IIOPubMessage) => boolean | Thenable<boolean>): void {
+		// no-op
+	}
+}