Improve cell language detection and add support for language magics (#4081)

* Move to using notebook language by default, with override in cell
* Update cell language on kernel change
* Tweak language logic so that it prefers code mirror mode, then falls back since this was failing some notebooks
* Add new package.json contribution to define language magics. These result in cell language changing. Language is cleared out on removing the language magic
* Added support for executing Python, R and Java in the SQL Kernel to prove this out. It converts to the sp_execute_external_script format

TODO in future PR:

* Need to hook up completion item support for magics (issue #4078)
* Should add indicator at the bottom of a cell when an alternate language has been detected (issue #4079)
* On executing Python, R or Java, should add some output showing the generated code (issue #4080)

src/sql/parts/notebook/models/cell.ts

@@ -49,7 +49,6 @@ export class CellModel implements ICellModel {
 			this._source = '';
 		}
 		this._isEditMode = this._cellType !== CellTypes.Markdown;
-		this.ensureDefaultLanguage();
 		if (_options && _options.isTrusted) {
 			this._isTrusted = true;
 		} else {
@@ -150,10 +149,16 @@ export class CellModel implements ICellModel {
 	}
 
 	public get language(): string {
-		return this._language;
+		if (this._cellType === CellTypes.Markdown) {
+			return 'markdown';
+		}
+		if (this._language) {
+			return this._language;
+		}
+		return this.options.notebook.language;
 	}
 
-	public set language(newLanguage: string) {
+	public setOverrideLanguage(newLanguage: string) {
 		this._language = newLanguage;
 	}
 
@@ -203,7 +208,7 @@ export class CellModel implements ICellModel {
 					}, false);
 					this.setFuture(future as FutureInternal);
 					// For now, await future completion. Later we should just track and handle cancellation based on model notifications
-					let result: nb.IExecuteReplyMsg = <nb.IExecuteReplyMsg><any> await future.done;
+					let result: nb.IExecuteReplyMsg = <nb.IExecuteReplyMsg><any>await future.done;
 					if (result && result.content) {
 						this.executionCount = result.content.execution_count;
 						if (result.content.status !== 'ok') {
@@ -254,7 +259,7 @@ export class CellModel implements ICellModel {
 
 	private sendNotification(notificationService: INotificationService, severity: Severity, message: string): void {
 		if (notificationService) {
-			notificationService.notify({ severity: severity, message: message});
+			notificationService.notify({ severity: severity, message: message });
 		}
 	}
 
@@ -382,7 +387,7 @@ export class CellModel implements ICellModel {
 					}
 				}
 			}
-			catch (e) {}
+			catch (e) { }
 		}
 		return output;
 	}
@@ -401,7 +406,7 @@ export class CellModel implements ICellModel {
 		};
 		if (this._cellType === CellTypes.Code) {
 			cellJson.metadata.language = this._language,
-			cellJson.outputs = this._outputs;
+				cellJson.outputs = this._outputs;
 			cellJson.execution_count = this.executionCount;
 		}
 		return cellJson as nb.ICellContents;
@@ -437,77 +442,15 @@ export class CellModel implements ICellModel {
 		this._outputs.push(output);
 	}
 
-  /**
-   * Normalize an output.
-   */
-  private _normalize(value: nb.ICellOutput): void {
-	if (notebookUtils.isStream(value)) {
-	  if (Array.isArray(value.text)) {
-		value.text = (value.text as string[]).join('\n');
-	  }
-	}
-  }
-
-	private get languageInfo(): nb.ILanguageInfo {
-		if (this._options && this._options.notebook && this._options.notebook.languageInfo) {
-			return this._options.notebook.languageInfo;
-		}
-		return undefined;
-	}
-
 	/**
-	 * Ensures there is a default language set, if none was already defined.
-	 * Will read information from the overall Notebook (passed as options to the model), or
-	 * if all else fails default back to python.
-	 *
+	 * Normalize an output.
 	 */
-	private ensureDefaultLanguage(): void {
-		// See if language is already set / is known based on cell type
-		if (this.hasLanguage()) {
-			return;
-		}
-		if (this._cellType === CellTypes.Markdown) {
-			this._language = 'markdown';
-			return;
-		}
-
-		// try set it based on overall Notebook language
-		this.trySetLanguageFromLangInfo();
-
-		// fallback to python
-		if (!this._language) {
-			this._language = 'python';
-		}
-	}
-
-	private trySetLanguageFromLangInfo() {
-		// In languageInfo, set the language to the "name" property
-		// If the "name" property isn't defined, check the "mimeType" property
-		// Otherwise, default to python as the language
-		let languageInfo = this.languageInfo;
-		if (languageInfo) {
-			if (languageInfo.name) {
-				this._language = languageInfo.name;
-			} else if (languageInfo.codemirror_mode) {
-				let codeMirrorMode: nb.ICodeMirrorMode = <nb.ICodeMirrorMode>(languageInfo.codemirror_mode);
-				if (codeMirrorMode && codeMirrorMode.name) {
-					this._language = codeMirrorMode.name;
-				}
-			} else if (languageInfo.mimetype) {
-				this._language = languageInfo.mimetype;
+	private _normalize(value: nb.ICellOutput): void {
+		if (notebookUtils.isStream(value)) {
+			if (Array.isArray(value.text)) {
+				value.text = (value.text as string[]).join('\n');
 			}
 		}
-
-		if (this._language) {
-			let mimeTypePrefix = 'x-';
-			if (this._language.includes(mimeTypePrefix)) {
-				this._language = this._language.replace(mimeTypePrefix, '');
-			}
-		}
-	}
-
-	private hasLanguage(): boolean {
-		return !!this._language;
 	}
 
 	private createUri(): void {

src/sql/parts/notebook/models/cellMagicMapper.ts

@@ -0,0 +1,55 @@
+
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the Source EULA. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { ICellMagicMapper, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
+
+const defaultKernel = '*';
+export class CellMagicMapper implements ICellMagicMapper {
+	private kernelToMagicMap = new Map<string,ILanguageMagic[]>();
+
+	constructor(languageMagics: ILanguageMagic[]) {
+		if (languageMagics) {
+			for (let magic of languageMagics) {
+				if (!magic.kernels || magic.kernels.length === 0) {
+					this.addKernelMapping(defaultKernel, magic);
+				}
+				if (magic.kernels) {
+					for (let kernel of magic.kernels) {
+						this.addKernelMapping(kernel.toLowerCase(), magic);
+					}
+				}
+			}
+		}
+	}
+
+	private addKernelMapping(kernelId: string, magic: ILanguageMagic): void {
+		let magics = this.kernelToMagicMap.get(kernelId) || [];
+		magics.push(magic);
+		this.kernelToMagicMap.set(kernelId, magics);
+	}
+
+	private findMagicForKernel(searchText: string, kernelId: string): ILanguageMagic | undefined {
+		if (kernelId === undefined || !searchText) {
+			return undefined;
+		}
+		searchText = searchText.toLowerCase();
+		let kernelMagics = this.kernelToMagicMap.get(kernelId) || [];
+		if (kernelMagics) {
+			return kernelMagics.find(m => m.magic.toLowerCase() === searchText);
+		}
+		return undefined;
+	}
+
+	toLanguageMagic(magic: string, kernelId: string): ILanguageMagic {
+		let languageMagic = this.findMagicForKernel(magic, kernelId.toLowerCase());
+		if (!languageMagic) {
+			languageMagic = this.findMagicForKernel(magic, defaultKernel);
+		}
+		return languageMagic;
+	}
+}

src/sql/parts/notebook/models/modelInterfaces.ts

@@ -267,9 +267,13 @@ export interface INotebookModel {
 	 */
 	readonly clientSession: IClientSession;
 	/**
-	 * LanguageInfo saved in the query book
+	 * LanguageInfo saved in the notebook
 	 */
 	readonly languageInfo: nb.ILanguageInfo;
+	/**
+	 * Current default language for the notebook
+	 */
+	readonly language: string;
 
 	/**
 	 * All notebook managers applicable for a given notebook
@@ -421,7 +425,7 @@ export enum CellExecutionState {
 export interface ICellModel {
 	cellUri: URI;
 	id: string;
-	language: string;
+	readonly language: string;
 	source: string;
 	cellType: CellType;
 	trustedMode: boolean;
@@ -435,6 +439,7 @@ export interface ICellModel {
 	setFuture(future: FutureInternal): void;
 	readonly executionState: CellExecutionState;
 	runCell(notificationService?: INotificationService): Promise<boolean>;
+	setOverrideLanguage(language: string);
 	equals(cellModel: ICellModel): boolean;
 	toJSON(): nb.ICellContents;
 }
@@ -465,6 +470,7 @@ export interface INotebookModelOptions {
 	providerId: string;
 	standardKernels: IStandardKernelWithProvider[];
 	defaultKernel: nb.IKernelSpec;
+	cellMagicMapper: ICellMagicMapper;
 
 	layoutChanged: Event<void>;
 
@@ -473,6 +479,22 @@ export interface INotebookModelOptions {
 	capabilitiesService: ICapabilitiesService;
 }
 
+export interface ILanguageMagic {
+	magic: string;
+	language: string;
+	kernels?: string[];
+	executionTarget?: string;
+}
+
+export interface ICellMagicMapper {
+	/**
+	 * Tries to find a language mapping for an identified cell magic
+	 * @param magic a string defining magic. For example for %%sql the magic text is sql
+	 * @param kernelId the name of the current kernel to use when looking up magics
+	 */
+	toLanguageMagic(magic: string, kernelId: string): ILanguageMagic | undefined;
+}
+
 export namespace notebookConstants {
 	export const SQL = 'SQL';
 }

src/sql/parts/notebook/models/notebookContexts.ts

@@ -8,7 +8,7 @@
 import { nb } from 'sqlops';
 
 import { localize } from 'vs/nls';
-import { IDefaultConnection, notebookConstants, INotebookModelOptions } from 'sql/parts/notebook/models/modelInterfaces';
+import { IDefaultConnection, notebookConstants } from 'sql/parts/notebook/models/modelInterfaces';
 import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement';
 import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';

src/sql/parts/notebook/models/notebookModel.ts

@@ -56,6 +56,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 
 	private _cells: ICellModel[];
 	private _defaultLanguageInfo: nb.ILanguageInfo;
+	private _language: string;
 	private _onErrorEmitter = new Emitter<INotification>();
 	private _savedKernelInfo: nb.IKernelInfo;
 	private readonly _nbformat: number = nbversion.MAJOR_VERSION;
@@ -68,31 +69,31 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private _kernelDisplayNameToNotebookProviderIds: Map<string, string> = new Map<string, string>();
 	private _onValidConnectionSelected = new Emitter<boolean>();
 
-	constructor(public notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) {
+	constructor(private _notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) {
 		super();
-		if (!notebookOptions || !notebookOptions.notebookUri || !notebookOptions.notebookManagers) {
+		if (!_notebookOptions || !_notebookOptions.notebookUri || !_notebookOptions.notebookManagers) {
 			throw new Error('path or notebook service not defined');
 		}
 		if (startSessionImmediately) {
 			this.backgroundStartSession();
 		}
 		this._trustedMode = false;
-		this._providerId = notebookOptions.providerId;
+		this._providerId = _notebookOptions.providerId;
 		this._onProviderIdChanged.fire(this._providerId);
-		this.notebookOptions.standardKernels.forEach(kernel => {
+		this._notebookOptions.standardKernels.forEach(kernel => {
 			this._kernelDisplayNameToConnectionProviderIds.set(kernel.name, kernel.connectionProviderIds);
 			this._kernelDisplayNameToNotebookProviderIds.set(kernel.name, kernel.notebookProvider);
 		});
-		if (this.notebookOptions.layoutChanged) {
-			this.notebookOptions.layoutChanged(() => this._layoutChanged.fire());
+		if (this._notebookOptions.layoutChanged) {
+			this._notebookOptions.layoutChanged(() => this._layoutChanged.fire());
 		}
-		this._defaultKernel = notebookOptions.defaultKernel;
+		this._defaultKernel = _notebookOptions.defaultKernel;
 	}
 
 	public get notebookManagers(): INotebookManager[] {
-		let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
+		let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
 		if (!notebookManagers.length) {
-			return this.notebookOptions.notebookManagers;
+			return this._notebookOptions.notebookManagers;
 		}
 		return notebookManagers;
 	}
@@ -107,11 +108,15 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		return manager;
 	}
 
+	public get notebookOptions(): INotebookModelOptions {
+		return this._notebookOptions;
+	}
+
 	public get notebookUri(): URI {
-		return this.notebookOptions.notebookUri;
+		return this._notebookOptions.notebookUri;
 	}
 	public set notebookUri(value: URI) {
-		this.notebookOptions.notebookUri = value;
+		this._notebookOptions.notebookUri = value;
 	}
 
 	public get hasServerManager(): boolean {
@@ -246,11 +251,11 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		try {
 			this._trustedMode = isTrusted;
 			let contents = null;
-			if (this.notebookOptions.notebookUri.scheme !== Schemas.untitled) {
+			if (this._notebookOptions.notebookUri.scheme !== Schemas.untitled) {
 				// TODO: separate ContentManager from NotebookManager
-				contents = await this.notebookManagers[0].contentManager.getNotebookContents(this.notebookOptions.notebookUri);
+				contents = await this.notebookManagers[0].contentManager.getNotebookContents(this._notebookOptions.notebookUri);
 			}
-			let factory = this.notebookOptions.factory;
+			let factory = this._notebookOptions.factory;
 			// if cells already exist, create them with language info (if it is saved)
 			this._cells = [];
 			this._defaultLanguageInfo = {
@@ -268,6 +273,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 					this._cells = contents.cells.map(c => factory.createCell(c, { notebook: this, isTrusted: isTrusted }));
 				}
 			}
+			this.trySetLanguageFromLangInfo();
 		} catch (error) {
 			this._inErrorState = true;
 			throw error;
@@ -317,7 +323,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			metadata: {},
 			execution_count: undefined
 		};
-		return this.notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true });
+		return this._notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true });
 	}
 
 	deleteCell(cellModel: ICellModel): void {
@@ -347,7 +353,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			if (edit.cell) {
 				// TODO: should we validate and complete required missing parameters?
 				let contents: nb.ICellContents = edit.cell as nb.ICellContents;
-				newCells.push(this.notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode }));
+				newCells.push(this._notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode }));
 			}
 			this._cells.splice(edit.range.start, edit.range.end - edit.range.start, ...newCells);
 			if (newCells.length > 0) {
@@ -374,16 +380,16 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	public backgroundStartSession(): void {
 		// TODO: only one session should be active at a time, depending on the current provider
 		this.notebookManagers.forEach(manager => {
-			let clientSession = this.notebookOptions.factory.createClientSession({
-				notebookUri: this.notebookOptions.notebookUri,
+			let clientSession = this._notebookOptions.factory.createClientSession({
+				notebookUri: this._notebookOptions.notebookUri,
 				notebookManager: manager,
-				notificationService: this.notebookOptions.notificationService
+				notificationService: this._notebookOptions.notificationService
 			});
 			this._clientSessions.push(clientSession);
 			if (!this._activeClientSession) {
 				this._activeClientSession = clientSession;
 			}
-			let profile = new ConnectionProfile(this.notebookOptions.capabilitiesService, this.connectionProfile);
+			let profile = new ConnectionProfile(this._notebookOptions.capabilitiesService, this.connectionProfile);
 
 			if (this.isValidConnection(profile)) {
 				this._activeConnection = profile;
@@ -405,7 +411,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	}
 
 	private isValidConnection(profile: IConnectionProfile | connection.Connection) {
-		let standardKernels = this.notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name);
+		let standardKernels = this._notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name);
 		let connectionProviderIds = standardKernels ? standardKernels.connectionProviderIds : undefined;
 		return profile && connectionProviderIds && connectionProviderIds.find(provider => provider === profile.providerName) !== undefined;
 	}
@@ -414,12 +420,51 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		return this._defaultLanguageInfo;
 	}
 
+	public get language(): string {
+		return this._language;
+	}
+
 	private updateLanguageInfo(info: nb.ILanguageInfo) {
 		if (info) {
 			this._defaultLanguageInfo = info;
+			this.trySetLanguageFromLangInfo();
 		}
 	}
 
+	private trySetLanguageFromLangInfo() {
+		// In languageInfo, set the language to the "name" property
+		// If the "name" property isn't defined, check the "mimeType" property
+		// Otherwise, default to python as the language
+		let languageInfo = this.languageInfo;
+		let language: string;
+		if (languageInfo) {
+			if (languageInfo.codemirror_mode) {
+				let codeMirrorMode: nb.ICodeMirrorMode = <nb.ICodeMirrorMode>(languageInfo.codemirror_mode);
+				if (codeMirrorMode && codeMirrorMode.name) {
+					language = codeMirrorMode.name;
+				}
+			}
+			if (!language && languageInfo.name) {
+				language = languageInfo.name;
+			}
+			if (!language && languageInfo.mimetype) {
+				language = languageInfo.mimetype;
+			}
+		}
+
+		if (language) {
+			let mimeTypePrefix = 'x-';
+			if (language.includes(mimeTypePrefix)) {
+				language = language.replace(mimeTypePrefix, '');
+			} else if (language.toLowerCase() === 'ipython') {
+				// Special case ipython because in many cases this is defined as the code mirror mode for python notebooks
+				language = 'python';
+			}
+		}
+
+		this._language = language;
+	}
+
 	public changeKernel(displayName: string): void {
 		let spec = this.getKernelSpecFromDisplayName(displayName);
 		this.doChangeKernel(spec);
@@ -457,7 +502,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			if (!newConnection && (this._activeContexts.defaultConnection.serverName === server)) {
 				newConnection = this._activeContexts.defaultConnection;
 			}
-			let newConnectionProfile = new ConnectionProfile(this.notebookOptions.capabilitiesService, newConnection);
+			let newConnectionProfile = new ConnectionProfile(this._notebookOptions.capabilitiesService, newConnection);
 			this._activeConnection = newConnectionProfile;
 			this.refreshConnections(newConnectionProfile);
 			this._activeClientSession.updateConnection(this._activeConnection.toIConnectionProfile()).then(
@@ -593,7 +638,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private async loadActiveContexts(kernelChangedArgs: nb.IKernelChangedArgs): Promise<void> {
 		if (kernelChangedArgs && kernelChangedArgs.newValue && kernelChangedArgs.newValue.name) {
 			let kernelDisplayName = this.getDisplayNameFromSpecName(kernelChangedArgs.newValue);
-			this._activeContexts = await NotebookContexts.getContextsForKernel(this.notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile);
+			this._activeContexts = await NotebookContexts.getContextsForKernel(this._notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile);
 			this._contextsChangedEmitter.fire();
 			if (this.contexts.defaultConnection !== undefined && this.contexts.defaultConnection.serverName !== undefined) {
 				await this.changeContext(this.contexts.defaultConnection.serverName);
@@ -622,7 +667,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			return false;
 		}
 		// TODO: refactor ContentManager out from NotebookManager
-		await this.notebookManagers[0].contentManager.save(this.notebookOptions.notebookUri, notebook);
+		await this.notebookManagers[0].contentManager.save(this._notebookOptions.notebookUri, notebook);
 		this._contentChangedEmitter.fire({
 			changeType: NotebookChangeType.DirtyStateChanged,
 			isDirty: false
@@ -653,9 +698,9 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private setProviderIdForKernel(kernelSpec: nb.IKernelSpec): void {
 		if (!kernelSpec) {
 			// Just use the 1st non-default provider, we don't have a better heuristic
-			let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
+			let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
 			if (!notebookManagers.length) {
-				notebookManagers = this.notebookOptions.notebookManagers;
+				notebookManagers = this._notebookOptions.notebookManagers;
 			}
 			if (notebookManagers.length > 0) {
 				this._providerId = notebookManagers[0].providerId;