Improve cell language detection and add support for language magics (#4081)

* Move to using notebook language by default, with override in cell
* Update cell language on kernel change
* Tweak language logic so that it prefers code mirror mode, then falls back since this was failing some notebooks
* Add new package.json contribution to define language magics. These result in cell language changing. Language is cleared out on removing the language magic
* Added support for executing Python, R and Java in the SQL Kernel to prove this out. It converts to the sp_execute_external_script format

TODO in future PR:

* Need to hook up completion item support for magics (issue #4078)
* Should add indicator at the bottom of a cell when an alternate language has been detected (issue #4079)
* On executing Python, R or Java, should add some output showing the generated code (issue #4080)

src/sql/parts/notebook/cellViews/code.component.ts

@@ -6,7 +6,6 @@ import 'vs/css!./code';
 
 import { OnInit, Component, Input, Inject, forwardRef, ElementRef, ChangeDetectorRef, ViewChild, Output, EventEmitter, OnChanges, SimpleChange } from '@angular/core';
 
-import { CommonServiceInterface } from 'sql/services/common/commonServiceInterface.service';
 import { AngularDisposable } from 'sql/base/node/lifecycle';
 import { QueryTextEditor } from 'sql/parts/modelComponents/queryTextEditor';
 import { CellToggleMoreActions } from 'sql/parts/notebook/cellToggleMoreActions';
@@ -26,12 +25,12 @@ import { UntitledEditorInput } from 'vs/workbench/common/editor/untitledEditorIn
 import * as DOM from 'vs/base/browser/dom';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
-import { IContextMenuService, IContextViewService } from 'vs/platform/contextview/browser/contextView';
-import { INotificationService } from 'vs/platform/notification/common/notification';
+import { IContextMenuService } from 'vs/platform/contextview/browser/contextView';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { Emitter, debounceEvent } from 'vs/base/common/event';
 import { CellTypes } from 'sql/parts/notebook/models/contracts';
 import { OVERRIDE_EDITOR_THEMING_SETTING } from 'sql/workbench/services/notebook/common/notebookService';
+import * as notebookUtils from 'sql/parts/notebook/notebookUtils';
 
 export const CODE_SELECTOR: string = 'code-component';
 const MARKDOWN_CLASS = 'markdown';
@@ -61,6 +60,12 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange
 
 	@Input() set model(value: NotebookModel) {
 		this._model = value;
+		this._register(value.kernelChanged(() => {
+			// On kernel change, need to reevaluate the language for each cell
+			// Refresh based on the cell magic (since this is kernel-dependent) and then update using notebook language
+			this.checkForLanguageMagics();
+			this.updateLanguageMode();
+		}));
 	}
 
 	@Input() set activeCellId(value: string) {
@@ -88,21 +93,17 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange
 	private _layoutEmitter = new Emitter<void>();
 
 	constructor(
-		@Inject(forwardRef(() => CommonServiceInterface)) private _bootstrapService: CommonServiceInterface,
-		@Inject(forwardRef(() => ChangeDetectorRef)) private _changeRef: ChangeDetectorRef,
 		@Inject(IWorkbenchThemeService) private themeService: IWorkbenchThemeService,
 		@Inject(IInstantiationService) private _instantiationService: IInstantiationService,
 		@Inject(IModelService) private _modelService: IModelService,
 		@Inject(IModeService) private _modeService: IModeService,
 		@Inject(IContextMenuService) private contextMenuService: IContextMenuService,
-		@Inject(IContextViewService) private contextViewService: IContextViewService,
-		@Inject(INotificationService) private notificationService: INotificationService,
 		@Inject(IConfigurationService) private _configurationService: IConfigurationService
 	) {
 		super();
 		this._cellToggleMoreActions = this._instantiationService.createInstance(CellToggleMoreActions);
-		debounceEvent(this._layoutEmitter.event, (l, e) => e, 250, /*leading=*/false)
-		(() => this.layout());
+		this._register(debounceEvent(this._layoutEmitter.event, (l, e) => e, 250, /*leading=*/false)
+		(() => this.layout()));
 
 	}
 
@@ -180,6 +181,7 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange
 			this._editor.setHeightToScrollHeight();
 			this.cellModel.source = this._editorModel.getValue();
 			this.onContentChanged.emit();
+			this.checkForLanguageMagics();
 			// TODO see if there's a better way to handle reassessing size.
 			setTimeout(() => this._layoutEmitter.fire(), 250);
 		}));
@@ -220,7 +222,31 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange
 		}
 	}
 
-	private updateLanguageMode() {
+	private checkForLanguageMagics(): void {
+		try {
+			if (!this.cellModel || this.cellModel.cellType !== CellTypes.Code) {
+				return;
+			}
+			if (this._editorModel && this._editor && this._editorModel.getLineCount() > 1) {
+				// Only try to match once we've typed past the first line
+				let magicName = notebookUtils.tryMatchCellMagic(this._editorModel.getLineContent(1));
+				if (magicName) {
+					let kernelName = this._model.clientSession && this._model.clientSession.kernel ? this._model.clientSession.kernel.name : undefined;
+					let magic = this._model.notebookOptions.cellMagicMapper.toLanguageMagic(magicName, kernelName);
+					if (magic && this.cellModel.language !== magic.language) {
+						this.cellModel.setOverrideLanguage(magic.language);
+						this.updateLanguageMode();
+					}
+				} else {
+					this.cellModel.setOverrideLanguage(undefined);
+				}
+			}
+		} catch (err) {
+			// No-op for now. Should we log?
+		}
+	}
+
+	private updateLanguageMode(): void {
 		if (this._editorModel && this._editor) {
 			this._modeService.getOrCreateMode(this.cellModel.language).then((modeValue) => {
 				this._modelService.setMode(this._editorModel, modeValue);

src/sql/parts/notebook/models/cell.ts

@@ -49,7 +49,6 @@ export class CellModel implements ICellModel {
 			this._source = '';
 		}
 		this._isEditMode = this._cellType !== CellTypes.Markdown;
-		this.ensureDefaultLanguage();
 		if (_options && _options.isTrusted) {
 			this._isTrusted = true;
 		} else {
@@ -150,10 +149,16 @@ export class CellModel implements ICellModel {
 	}
 
 	public get language(): string {
-		return this._language;
+		if (this._cellType === CellTypes.Markdown) {
+			return 'markdown';
+		}
+		if (this._language) {
+			return this._language;
+		}
+		return this.options.notebook.language;
 	}
 
-	public set language(newLanguage: string) {
+	public setOverrideLanguage(newLanguage: string) {
 		this._language = newLanguage;
 	}
 
@@ -203,7 +208,7 @@ export class CellModel implements ICellModel {
 					}, false);
 					this.setFuture(future as FutureInternal);
 					// For now, await future completion. Later we should just track and handle cancellation based on model notifications
-					let result: nb.IExecuteReplyMsg = <nb.IExecuteReplyMsg><any> await future.done;
+					let result: nb.IExecuteReplyMsg = <nb.IExecuteReplyMsg><any>await future.done;
 					if (result && result.content) {
 						this.executionCount = result.content.execution_count;
 						if (result.content.status !== 'ok') {
@@ -254,7 +259,7 @@ export class CellModel implements ICellModel {
 
 	private sendNotification(notificationService: INotificationService, severity: Severity, message: string): void {
 		if (notificationService) {
-			notificationService.notify({ severity: severity, message: message});
+			notificationService.notify({ severity: severity, message: message });
 		}
 	}
 
@@ -382,7 +387,7 @@ export class CellModel implements ICellModel {
 					}
 				}
 			}
-			catch (e) {}
+			catch (e) { }
 		}
 		return output;
 	}
@@ -401,7 +406,7 @@ export class CellModel implements ICellModel {
 		};
 		if (this._cellType === CellTypes.Code) {
 			cellJson.metadata.language = this._language,
-			cellJson.outputs = this._outputs;
+				cellJson.outputs = this._outputs;
 			cellJson.execution_count = this.executionCount;
 		}
 		return cellJson as nb.ICellContents;
@@ -437,77 +442,15 @@ export class CellModel implements ICellModel {
 		this._outputs.push(output);
 	}
 
-  /**
-   * Normalize an output.
-   */
-  private _normalize(value: nb.ICellOutput): void {
-	if (notebookUtils.isStream(value)) {
-	  if (Array.isArray(value.text)) {
-		value.text = (value.text as string[]).join('\n');
-	  }
-	}
-  }
-
-	private get languageInfo(): nb.ILanguageInfo {
-		if (this._options && this._options.notebook && this._options.notebook.languageInfo) {
-			return this._options.notebook.languageInfo;
-		}
-		return undefined;
-	}
-
 	/**
-	 * Ensures there is a default language set, if none was already defined.
-	 * Will read information from the overall Notebook (passed as options to the model), or
-	 * if all else fails default back to python.
-	 *
+	 * Normalize an output.
 	 */
-	private ensureDefaultLanguage(): void {
-		// See if language is already set / is known based on cell type
-		if (this.hasLanguage()) {
-			return;
-		}
-		if (this._cellType === CellTypes.Markdown) {
-			this._language = 'markdown';
-			return;
-		}
-
-		// try set it based on overall Notebook language
-		this.trySetLanguageFromLangInfo();
-
-		// fallback to python
-		if (!this._language) {
-			this._language = 'python';
-		}
-	}
-
-	private trySetLanguageFromLangInfo() {
-		// In languageInfo, set the language to the "name" property
-		// If the "name" property isn't defined, check the "mimeType" property
-		// Otherwise, default to python as the language
-		let languageInfo = this.languageInfo;
-		if (languageInfo) {
-			if (languageInfo.name) {
-				this._language = languageInfo.name;
-			} else if (languageInfo.codemirror_mode) {
-				let codeMirrorMode: nb.ICodeMirrorMode = <nb.ICodeMirrorMode>(languageInfo.codemirror_mode);
-				if (codeMirrorMode && codeMirrorMode.name) {
-					this._language = codeMirrorMode.name;
-				}
-			} else if (languageInfo.mimetype) {
-				this._language = languageInfo.mimetype;
+	private _normalize(value: nb.ICellOutput): void {
+		if (notebookUtils.isStream(value)) {
+			if (Array.isArray(value.text)) {
+				value.text = (value.text as string[]).join('\n');
 			}
 		}
-
-		if (this._language) {
-			let mimeTypePrefix = 'x-';
-			if (this._language.includes(mimeTypePrefix)) {
-				this._language = this._language.replace(mimeTypePrefix, '');
-			}
-		}
-	}
-
-	private hasLanguage(): boolean {
-		return !!this._language;
 	}
 
 	private createUri(): void {

src/sql/parts/notebook/models/cellMagicMapper.ts

@@ -0,0 +1,55 @@
+
+/*---------------------------------------------------------------------------------------------
+*  Copyright (c) Microsoft Corporation. All rights reserved.
+*  Licensed under the Source EULA. See License.txt in the project root for license information.
+*--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { ICellMagicMapper, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
+
+const defaultKernel = '*';
+export class CellMagicMapper implements ICellMagicMapper {
+	private kernelToMagicMap = new Map<string,ILanguageMagic[]>();
+
+	constructor(languageMagics: ILanguageMagic[]) {
+		if (languageMagics) {
+			for (let magic of languageMagics) {
+				if (!magic.kernels || magic.kernels.length === 0) {
+					this.addKernelMapping(defaultKernel, magic);
+				}
+				if (magic.kernels) {
+					for (let kernel of magic.kernels) {
+						this.addKernelMapping(kernel.toLowerCase(), magic);
+					}
+				}
+			}
+		}
+	}
+
+	private addKernelMapping(kernelId: string, magic: ILanguageMagic): void {
+		let magics = this.kernelToMagicMap.get(kernelId) || [];
+		magics.push(magic);
+		this.kernelToMagicMap.set(kernelId, magics);
+	}
+
+	private findMagicForKernel(searchText: string, kernelId: string): ILanguageMagic | undefined {
+		if (kernelId === undefined || !searchText) {
+			return undefined;
+		}
+		searchText = searchText.toLowerCase();
+		let kernelMagics = this.kernelToMagicMap.get(kernelId) || [];
+		if (kernelMagics) {
+			return kernelMagics.find(m => m.magic.toLowerCase() === searchText);
+		}
+		return undefined;
+	}
+
+	toLanguageMagic(magic: string, kernelId: string): ILanguageMagic {
+		let languageMagic = this.findMagicForKernel(magic, kernelId.toLowerCase());
+		if (!languageMagic) {
+			languageMagic = this.findMagicForKernel(magic, defaultKernel);
+		}
+		return languageMagic;
+	}
+}

src/sql/parts/notebook/models/modelInterfaces.ts

@@ -267,9 +267,13 @@ export interface INotebookModel {
 	 */
 	readonly clientSession: IClientSession;
 	/**
-	 * LanguageInfo saved in the query book
+	 * LanguageInfo saved in the notebook
 	 */
 	readonly languageInfo: nb.ILanguageInfo;
+	/**
+	 * Current default language for the notebook
+	 */
+	readonly language: string;
 
 	/**
 	 * All notebook managers applicable for a given notebook
@@ -421,7 +425,7 @@ export enum CellExecutionState {
 export interface ICellModel {
 	cellUri: URI;
 	id: string;
-	language: string;
+	readonly language: string;
 	source: string;
 	cellType: CellType;
 	trustedMode: boolean;
@@ -435,6 +439,7 @@ export interface ICellModel {
 	setFuture(future: FutureInternal): void;
 	readonly executionState: CellExecutionState;
 	runCell(notificationService?: INotificationService): Promise<boolean>;
+	setOverrideLanguage(language: string);
 	equals(cellModel: ICellModel): boolean;
 	toJSON(): nb.ICellContents;
 }
@@ -465,6 +470,7 @@ export interface INotebookModelOptions {
 	providerId: string;
 	standardKernels: IStandardKernelWithProvider[];
 	defaultKernel: nb.IKernelSpec;
+	cellMagicMapper: ICellMagicMapper;
 
 	layoutChanged: Event<void>;
 
@@ -473,6 +479,22 @@ export interface INotebookModelOptions {
 	capabilitiesService: ICapabilitiesService;
 }
 
+export interface ILanguageMagic {
+	magic: string;
+	language: string;
+	kernels?: string[];
+	executionTarget?: string;
+}
+
+export interface ICellMagicMapper {
+	/**
+	 * Tries to find a language mapping for an identified cell magic
+	 * @param magic a string defining magic. For example for %%sql the magic text is sql
+	 * @param kernelId the name of the current kernel to use when looking up magics
+	 */
+	toLanguageMagic(magic: string, kernelId: string): ILanguageMagic | undefined;
+}
+
 export namespace notebookConstants {
 	export const SQL = 'SQL';
 }

src/sql/parts/notebook/models/notebookContexts.ts

@@ -8,7 +8,7 @@
 import { nb } from 'sqlops';
 
 import { localize } from 'vs/nls';
-import { IDefaultConnection, notebookConstants, INotebookModelOptions } from 'sql/parts/notebook/models/modelInterfaces';
+import { IDefaultConnection, notebookConstants } from 'sql/parts/notebook/models/modelInterfaces';
 import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement';
 import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';

src/sql/parts/notebook/models/notebookModel.ts

@@ -56,6 +56,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 
 	private _cells: ICellModel[];
 	private _defaultLanguageInfo: nb.ILanguageInfo;
+	private _language: string;
 	private _onErrorEmitter = new Emitter<INotification>();
 	private _savedKernelInfo: nb.IKernelInfo;
 	private readonly _nbformat: number = nbversion.MAJOR_VERSION;
@@ -68,31 +69,31 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private _kernelDisplayNameToNotebookProviderIds: Map<string, string> = new Map<string, string>();
 	private _onValidConnectionSelected = new Emitter<boolean>();
 
-	constructor(public notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) {
+	constructor(private _notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) {
 		super();
-		if (!notebookOptions || !notebookOptions.notebookUri || !notebookOptions.notebookManagers) {
+		if (!_notebookOptions || !_notebookOptions.notebookUri || !_notebookOptions.notebookManagers) {
 			throw new Error('path or notebook service not defined');
 		}
 		if (startSessionImmediately) {
 			this.backgroundStartSession();
 		}
 		this._trustedMode = false;
-		this._providerId = notebookOptions.providerId;
+		this._providerId = _notebookOptions.providerId;
 		this._onProviderIdChanged.fire(this._providerId);
-		this.notebookOptions.standardKernels.forEach(kernel => {
+		this._notebookOptions.standardKernels.forEach(kernel => {
 			this._kernelDisplayNameToConnectionProviderIds.set(kernel.name, kernel.connectionProviderIds);
 			this._kernelDisplayNameToNotebookProviderIds.set(kernel.name, kernel.notebookProvider);
 		});
-		if (this.notebookOptions.layoutChanged) {
-			this.notebookOptions.layoutChanged(() => this._layoutChanged.fire());
+		if (this._notebookOptions.layoutChanged) {
+			this._notebookOptions.layoutChanged(() => this._layoutChanged.fire());
 		}
-		this._defaultKernel = notebookOptions.defaultKernel;
+		this._defaultKernel = _notebookOptions.defaultKernel;
 	}
 
 	public get notebookManagers(): INotebookManager[] {
-		let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
+		let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
 		if (!notebookManagers.length) {
-			return this.notebookOptions.notebookManagers;
+			return this._notebookOptions.notebookManagers;
 		}
 		return notebookManagers;
 	}
@@ -107,11 +108,15 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		return manager;
 	}
 
+	public get notebookOptions(): INotebookModelOptions {
+		return this._notebookOptions;
+	}
+
 	public get notebookUri(): URI {
-		return this.notebookOptions.notebookUri;
+		return this._notebookOptions.notebookUri;
 	}
 	public set notebookUri(value: URI) {
-		this.notebookOptions.notebookUri = value;
+		this._notebookOptions.notebookUri = value;
 	}
 
 	public get hasServerManager(): boolean {
@@ -246,11 +251,11 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		try {
 			this._trustedMode = isTrusted;
 			let contents = null;
-			if (this.notebookOptions.notebookUri.scheme !== Schemas.untitled) {
+			if (this._notebookOptions.notebookUri.scheme !== Schemas.untitled) {
 				// TODO: separate ContentManager from NotebookManager
-				contents = await this.notebookManagers[0].contentManager.getNotebookContents(this.notebookOptions.notebookUri);
+				contents = await this.notebookManagers[0].contentManager.getNotebookContents(this._notebookOptions.notebookUri);
 			}
-			let factory = this.notebookOptions.factory;
+			let factory = this._notebookOptions.factory;
 			// if cells already exist, create them with language info (if it is saved)
 			this._cells = [];
 			this._defaultLanguageInfo = {
@@ -268,6 +273,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 					this._cells = contents.cells.map(c => factory.createCell(c, { notebook: this, isTrusted: isTrusted }));
 				}
 			}
+			this.trySetLanguageFromLangInfo();
 		} catch (error) {
 			this._inErrorState = true;
 			throw error;
@@ -317,7 +323,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			metadata: {},
 			execution_count: undefined
 		};
-		return this.notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true });
+		return this._notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true });
 	}
 
 	deleteCell(cellModel: ICellModel): void {
@@ -347,7 +353,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			if (edit.cell) {
 				// TODO: should we validate and complete required missing parameters?
 				let contents: nb.ICellContents = edit.cell as nb.ICellContents;
-				newCells.push(this.notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode }));
+				newCells.push(this._notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode }));
 			}
 			this._cells.splice(edit.range.start, edit.range.end - edit.range.start, ...newCells);
 			if (newCells.length > 0) {
@@ -374,16 +380,16 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	public backgroundStartSession(): void {
 		// TODO: only one session should be active at a time, depending on the current provider
 		this.notebookManagers.forEach(manager => {
-			let clientSession = this.notebookOptions.factory.createClientSession({
-				notebookUri: this.notebookOptions.notebookUri,
+			let clientSession = this._notebookOptions.factory.createClientSession({
+				notebookUri: this._notebookOptions.notebookUri,
 				notebookManager: manager,
-				notificationService: this.notebookOptions.notificationService
+				notificationService: this._notebookOptions.notificationService
 			});
 			this._clientSessions.push(clientSession);
 			if (!this._activeClientSession) {
 				this._activeClientSession = clientSession;
 			}
-			let profile = new ConnectionProfile(this.notebookOptions.capabilitiesService, this.connectionProfile);
+			let profile = new ConnectionProfile(this._notebookOptions.capabilitiesService, this.connectionProfile);
 
 			if (this.isValidConnection(profile)) {
 				this._activeConnection = profile;
@@ -405,7 +411,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	}
 
 	private isValidConnection(profile: IConnectionProfile | connection.Connection) {
-		let standardKernels = this.notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name);
+		let standardKernels = this._notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name);
 		let connectionProviderIds = standardKernels ? standardKernels.connectionProviderIds : undefined;
 		return profile && connectionProviderIds && connectionProviderIds.find(provider => provider === profile.providerName) !== undefined;
 	}
@@ -414,12 +420,51 @@ export class NotebookModel extends Disposable implements INotebookModel {
 		return this._defaultLanguageInfo;
 	}
 
+	public get language(): string {
+		return this._language;
+	}
+
 	private updateLanguageInfo(info: nb.ILanguageInfo) {
 		if (info) {
 			this._defaultLanguageInfo = info;
+			this.trySetLanguageFromLangInfo();
 		}
 	}
 
+	private trySetLanguageFromLangInfo() {
+		// In languageInfo, set the language to the "name" property
+		// If the "name" property isn't defined, check the "mimeType" property
+		// Otherwise, default to python as the language
+		let languageInfo = this.languageInfo;
+		let language: string;
+		if (languageInfo) {
+			if (languageInfo.codemirror_mode) {
+				let codeMirrorMode: nb.ICodeMirrorMode = <nb.ICodeMirrorMode>(languageInfo.codemirror_mode);
+				if (codeMirrorMode && codeMirrorMode.name) {
+					language = codeMirrorMode.name;
+				}
+			}
+			if (!language && languageInfo.name) {
+				language = languageInfo.name;
+			}
+			if (!language && languageInfo.mimetype) {
+				language = languageInfo.mimetype;
+			}
+		}
+
+		if (language) {
+			let mimeTypePrefix = 'x-';
+			if (language.includes(mimeTypePrefix)) {
+				language = language.replace(mimeTypePrefix, '');
+			} else if (language.toLowerCase() === 'ipython') {
+				// Special case ipython because in many cases this is defined as the code mirror mode for python notebooks
+				language = 'python';
+			}
+		}
+
+		this._language = language;
+	}
+
 	public changeKernel(displayName: string): void {
 		let spec = this.getKernelSpecFromDisplayName(displayName);
 		this.doChangeKernel(spec);
@@ -457,7 +502,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			if (!newConnection && (this._activeContexts.defaultConnection.serverName === server)) {
 				newConnection = this._activeContexts.defaultConnection;
 			}
-			let newConnectionProfile = new ConnectionProfile(this.notebookOptions.capabilitiesService, newConnection);
+			let newConnectionProfile = new ConnectionProfile(this._notebookOptions.capabilitiesService, newConnection);
 			this._activeConnection = newConnectionProfile;
 			this.refreshConnections(newConnectionProfile);
 			this._activeClientSession.updateConnection(this._activeConnection.toIConnectionProfile()).then(
@@ -593,7 +638,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private async loadActiveContexts(kernelChangedArgs: nb.IKernelChangedArgs): Promise<void> {
 		if (kernelChangedArgs && kernelChangedArgs.newValue && kernelChangedArgs.newValue.name) {
 			let kernelDisplayName = this.getDisplayNameFromSpecName(kernelChangedArgs.newValue);
-			this._activeContexts = await NotebookContexts.getContextsForKernel(this.notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile);
+			this._activeContexts = await NotebookContexts.getContextsForKernel(this._notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile);
 			this._contextsChangedEmitter.fire();
 			if (this.contexts.defaultConnection !== undefined && this.contexts.defaultConnection.serverName !== undefined) {
 				await this.changeContext(this.contexts.defaultConnection.serverName);
@@ -622,7 +667,7 @@ export class NotebookModel extends Disposable implements INotebookModel {
 			return false;
 		}
 		// TODO: refactor ContentManager out from NotebookManager
-		await this.notebookManagers[0].contentManager.save(this.notebookOptions.notebookUri, notebook);
+		await this.notebookManagers[0].contentManager.save(this._notebookOptions.notebookUri, notebook);
 		this._contentChangedEmitter.fire({
 			changeType: NotebookChangeType.DirtyStateChanged,
 			isDirty: false
@@ -653,9 +698,9 @@ export class NotebookModel extends Disposable implements INotebookModel {
 	private setProviderIdForKernel(kernelSpec: nb.IKernelSpec): void {
 		if (!kernelSpec) {
 			// Just use the 1st non-default provider, we don't have a better heuristic
-			let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
+			let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER);
 			if (!notebookManagers.length) {
-				notebookManagers = this.notebookOptions.notebookManagers;
+				notebookManagers = this._notebookOptions.notebookManagers;
 			}
 			if (notebookManagers.length > 0) {
 				this._providerId = notebookManagers[0].providerId;

src/sql/parts/notebook/notebook.component.ts

@@ -49,6 +49,7 @@ import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/un
 import { IEditorGroupsService } from 'vs/workbench/services/group/common/editorGroupsService';
 import { IConnectionDialogService } from 'sql/workbench/services/connection/common/connectionDialogService';
 import { ICapabilitiesService } from 'sql/platform/capabilities/common/capabilitiesService';
+import { CellMagicMapper } from 'sql/parts/notebook/models/cellMagicMapper';
 
 export const NOTEBOOK_SELECTOR: string = 'notebook-component';
 
@@ -263,6 +264,7 @@ export class NotebookComponent extends AngularDisposable implements OnInit, OnDe
 			notificationService: this.notificationService,
 			notebookManagers: this.notebookManagers,
 			standardKernels: this._notebookParams.input.standardKernels,
+			cellMagicMapper: new CellMagicMapper(this.notebookService.languageMagics),
 			providerId: notebookUtils.sqlNotebooksEnabled(this.contextKeyService) ? 'sql' : 'jupyter', // this is tricky; really should also depend on the connection profile
 			defaultKernel: this._notebookParams.input.defaultKernel,
 			layoutChanged: this._notebookParams.input.layoutChanged,

src/sql/parts/notebook/notebookUtils.ts

@@ -104,4 +104,15 @@ export interface IStandardKernelWithProvider {
 	readonly name: string;
 	readonly connectionProviderIds: string[];
 	readonly notebookProvider: string;
+}
+
+export function tryMatchCellMagic(input: string): string {
+	if (!input) {
+		return input;
+	}
+	let firstLine = input.trimLeft();
+	let magicRegex = /^%%(\w+)/g;
+	let match = magicRegex.exec(firstLine);
+	let magicName = match && match[1];
+	return magicName;
 }

src/sql/workbench/services/notebook/common/notebookRegistry.ts

@@ -13,7 +13,8 @@ import * as sqlops from 'sqlops';
 import { Event, Emitter } from 'vs/base/common/event';
 
 export const Extensions = {
-	NotebookProviderContribution: 'notebook.providers'
+	NotebookProviderContribution: 'notebook.providers',
+	NotebookLanguageMagicContribution: 'notebook.languagemagics'
 };
 
 export interface NotebookProviderRegistration {
@@ -94,16 +95,67 @@ let notebookContrib: IJSONSchema = {
 		}
 	]
 };
+let notebookLanguageMagicType: IJSONSchema = {
+	type: 'object',
+	default: { magic: '', language: '', kernels: [], executionTarget: null },
+	properties: {
+		magic: {
+			description: localize('carbon.extension.contributes.notebook.magic', 'Name of the cell magic, such as "%%sql".'),
+			type: 'string'
+		},
+		language: {
+			description: localize('carbon.extension.contributes.notebook.language', 'The cell language to be used if this cell magic is included in the cell'),
+			type: 'string'
+		},
+		executionTarget: {
+			description: localize('carbon.extension.contributes.notebook.executionTarget', 'Optional execution target this magic indicates, for example Spark vs SQL'),
+			type: 'string'
+		},
+		kernels: {
+			description: localize('carbon.extension.contributes.notebook.kernels', 'Optional set of kernels this is valid for, e.g. python3, pyspark3, sql'),
+			oneOf: [
+				{ type: 'string' },
+				{
+					type: 'array',
+					items: {
+						type: 'string'
+					}
+				}
+			]
+		}
+	}
+};
+
+let languageMagicContrib: IJSONSchema = {
+	description: localize('vscode.extension.contributes.notebook.languagemagics', "Contributes notebook language."),
+	oneOf: [
+		notebookLanguageMagicType,
+		{
+			type: 'array',
+			items: notebookLanguageMagicType
+		}
+	]
+};
+
+export interface NotebookLanguageMagicRegistration {
+	magic: string;
+	language: string;
+	kernels?: string[];
+	executionTarget?: string;
+}
 
 export interface INotebookProviderRegistry {
-	readonly registrations: NotebookProviderRegistration[];
+	readonly providers: NotebookProviderRegistration[];
+	readonly languageMagics: NotebookLanguageMagicRegistration[];
 	readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }>;
 
-	registerNotebookProvider(registration: NotebookProviderRegistration): void;
+	registerNotebookProvider(provider: NotebookProviderRegistration): void;
+	registerNotebookLanguageMagic(magic: NotebookLanguageMagicRegistration): void;
 }
 
 class NotebookProviderRegistry implements INotebookProviderRegistry {
 	private providerIdToRegistration = new Map<string, NotebookProviderRegistration>();
+	private magicToRegistration = new Map<string, NotebookLanguageMagicRegistration>();
 	private _onNewRegistration = new Emitter<{ id: string, registration: NotebookProviderRegistration }>();
 	public readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }> = this._onNewRegistration.event;
 
@@ -114,11 +166,22 @@ class NotebookProviderRegistry implements INotebookProviderRegistry {
 		this._onNewRegistration.fire({ id: registration.provider, registration: registration });
 	}
 
-	public get registrations(): NotebookProviderRegistration[] {
+	public get providers(): NotebookProviderRegistration[] {
 		let registrationArray: NotebookProviderRegistration[] = [];
 		this.providerIdToRegistration.forEach(p => registrationArray.push(p));
 		return registrationArray;
 	}
+
+	registerNotebookLanguageMagic(magicRegistration: NotebookLanguageMagicRegistration): void {
+		this.magicToRegistration.set(magicRegistration.magic, magicRegistration);
+	}
+
+	public get languageMagics(): NotebookLanguageMagicRegistration[] {
+		let registrationArray: NotebookLanguageMagicRegistration[] = [];
+		this.magicToRegistration.forEach(p => registrationArray.push(p));
+		return registrationArray;
+	}
+
 }
 
 const notebookProviderRegistry = new NotebookProviderRegistry();
@@ -142,3 +205,21 @@ ExtensionsRegistry.registerExtensionPoint<NotebookProviderRegistration | Noteboo
 		}
 	}
 });
+
+ExtensionsRegistry.registerExtensionPoint<NotebookLanguageMagicRegistration | NotebookLanguageMagicRegistration[]>(Extensions.NotebookLanguageMagicContribution, [], languageMagicContrib).setHandler(extensions => {
+
+	function handleExtension(contrib: NotebookLanguageMagicRegistration, extension: IExtensionPointUser<any>) {
+		notebookProviderRegistry.registerNotebookLanguageMagic(contrib);
+	}
+
+	for (let extension of extensions) {
+		const { value } = extension;
+		if (Array.isArray<NotebookLanguageMagicRegistration>(value)) {
+			for (let command of value) {
+				handleExtension(command, extension);
+			}
+		} else {
+			handleExtension(value, extension);
+		}
+	}
+});

src/sql/workbench/services/notebook/common/notebookService.ts

@@ -16,7 +16,7 @@ import { ModelFactory } from 'sql/parts/notebook/models/modelFactory';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
 import { NotebookInput } from 'sql/parts/notebook/notebookInput';
 import { ISingleNotebookEditOperation } from 'sql/workbench/api/common/sqlExtHostTypes';
-import { ICellModel, INotebookModel } from 'sql/parts/notebook/models/modelInterfaces';
+import { ICellModel, INotebookModel, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 
 export const SERVICE_ID = 'notebookService';
 export const INotebookService = createDecorator<INotebookService>(SERVICE_ID);
@@ -35,6 +35,7 @@ export interface INotebookService {
 
 	readonly isRegistrationComplete: boolean;
 	readonly registrationComplete: Promise<void>;
+	readonly languageMagics: ILanguageMagic[];
 	/**
 	 * Register a metadata provider
 	 */

src/sql/workbench/services/notebook/common/notebookServiceImpl.ts

@@ -38,6 +38,7 @@ import { IEditorGroupsService } from 'vs/workbench/services/group/common/editorG
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IThemeService } from 'vs/platform/theme/common/themeService';
 import { registerNotebookThemes } from 'sql/parts/notebook/notebookStyles';
+import { ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 
 export interface NotebookProviderProperties {
 	provider: string;
@@ -333,6 +334,12 @@ export class NotebookService extends Disposable implements INotebookService {
 		}
 	}
 
+	get languageMagics(): ILanguageMagic[] {
+		return notebookRegistry.languageMagics;
+	}
+
+	// PRIVATE HELPERS /////////////////////////////////////////////////////
+
 	private sendNotebookCloseToProvider(editor: INotebookEditor): void {
 		let notebookUri = editor.notebookParams.notebookUri;
 		let uriString = notebookUri.toString();
@@ -347,7 +354,6 @@ export class NotebookService extends Disposable implements INotebookService {
 		}
 	}
 
-	// PRIVATE HELPERS /////////////////////////////////////////////////////
 	private async doWithProvider<T>(providerId: string, op: (provider: INotebookProvider) => Thenable<T>): Promise<T> {
 		// Make sure the provider exists before attempting to retrieve accounts
 		let provider: INotebookProvider = await this.getProviderInstance(providerId);
@@ -405,7 +411,7 @@ export class NotebookService extends Disposable implements INotebookService {
 	}
 
 	private cleanupProviders(): void {
-		let knownProviders = Object.keys(notebookRegistry.registrations);
+		let knownProviders = Object.keys(notebookRegistry.providers);
 		let cache = this.providersMemento.notebookProviderCache;
 		for (let key in cache) {
 			if (!knownProviders.includes(key)) {

src/sql/workbench/services/notebook/common/sqlSessionManager.ts

@@ -4,9 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+import * as os from 'os';
 import { nb, QueryExecuteSubsetResult, IDbColumn, BatchSummary, IResultMessage } from 'sqlops';
 import { localize } from 'vs/nls';
-import { FutureInternal } from 'sql/parts/notebook/models/modelInterfaces';
+import { FutureInternal, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 import QueryRunner, { EventType } from 'sql/platform/query/common/queryRunner';
 import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@@ -18,6 +19,7 @@ import { IErrorMessageService } from 'sql/platform/errorMessage/common/errorMess
 import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
 import { escape } from 'sql/base/common/strings';
+import * as notebookUtils from 'sql/parts/notebook/notebookUtils';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 
 export const sqlKernel: string = localize('sqlKernel', 'SQL');
@@ -26,12 +28,23 @@ export const MAX_ROWS = 5000;
 export const NotebookConfigSectionName = 'notebook';
 export const MaxTableRowsConfigName = 'maxTableRows';
 
-let sqlKernelSpec: nb.IKernelSpec = ({
+const sqlKernelSpec: nb.IKernelSpec = ({
 	name: sqlKernel,
 	language: 'sql',
 	display_name: sqlKernel
 });
 
+const languageMagics: ILanguageMagic[] = [{
+	language: 'Python',
+	magic: 'lang_python'
+}, {
+	language: 'R',
+	magic: 'lang_r'
+}, {
+	language: 'Java',
+	magic: 'lang_java'
+}];
+
 export interface SQLData {
 	columns: Array<string>;
 	rows: Array<Array<string>>;
@@ -135,12 +148,22 @@ class SqlKernel extends Disposable implements nb.IKernel {
 	private _id: string;
 	private _future: SQLFuture;
 	private _executionCount: number = 0;
+	private _magicToExecutorMap = new Map<string, ExternalScriptMagic>();
 
-	constructor( @IConnectionManagementService private _connectionManagementService: IConnectionManagementService,
+	constructor(@IConnectionManagementService private _connectionManagementService: IConnectionManagementService,
 		@IInstantiationService private _instantiationService: IInstantiationService,
 		@IErrorMessageService private _errorMessageService: IErrorMessageService,
-		@IConfigurationService private _configurationService: IConfigurationService) {
+		@IConfigurationService private _configurationService: IConfigurationService
+	) {
 		super();
+		this.initMagics();
+	}
+
+	private initMagics(): void {
+		for (let magic of languageMagics) {
+			let scriptMagic = new ExternalScriptMagic(magic.language);
+			this._magicToExecutorMap.set(magic.magic, scriptMagic);
+		}
 	}
 
 	public get id(): string {
@@ -197,6 +220,7 @@ class SqlKernel extends Disposable implements nb.IKernel {
 
 	requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture {
 		let canRun: boolean = true;
+		let code = this.getCodeWithoutCellMagic(content);
 		if (this._queryRunner) {
 			// Cancel any existing query
 			if (this._future && !this._queryRunner.hasCompleted) {
@@ -204,14 +228,13 @@ class SqlKernel extends Disposable implements nb.IKernel {
 				// TODO when we can just show error as an output, should show an "execution canceled" error in output
 				this._future.handleDone();
 			}
-			this._queryRunner.runQuery(content.code);
+			this._queryRunner.runQuery(code);
 		} else if (this._currentConnection) {
 			let connectionUri = Utils.generateUri(this._currentConnection, 'notebook');
 			this._queryRunner = this._instantiationService.createInstance(QueryRunner, connectionUri);
-			this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) =>
-			{
+			this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) => {
 				this.addQueryEventListeners(this._queryRunner);
-				this._queryRunner.runQuery(content.code);
+				this._queryRunner.runQuery(code);
 			});
 		} else {
 			canRun = false;
@@ -231,6 +254,25 @@ class SqlKernel extends Disposable implements nb.IKernel {
 		return this._future;
 	}
 
+	private getCodeWithoutCellMagic(content: nb.IExecuteRequest): string {
+		let code = content.code;
+		let firstLineEnd = code.indexOf(os.EOL);
+		let firstLine = code.substring(0, (firstLineEnd >= 0) ? firstLineEnd : 0).trimLeft();
+		if (firstLine.startsWith('%%')) {
+			// Strip out the line
+			code = code.substring(firstLineEnd, code.length);
+			// Try and match to an external script magic. If we add more magics later, should handle transforms better
+			let magic = notebookUtils.tryMatchCellMagic(firstLine);
+			if (magic) {
+				let executor = this._magicToExecutorMap.get(magic.toLowerCase());
+				if (executor) {
+					code = executor.convertToExternalScript(code);
+				}
+			}
+		}
+		return code;
+	}
+
 	requestComplete(content: nb.ICompleteRequest): Thenable<nb.ICompleteReplyMsg> {
 		let response: Partial<nb.ICompleteReplyMsg> = {};
 		return Promise.resolve(response as nb.ICompleteReplyMsg);
@@ -388,7 +430,7 @@ export class SQLFuture extends Disposable implements FutureInternal {
 	private convertToDataResource(columns: IDbColumn[], subsetResult: QueryExecuteSubsetResult): IDataResource {
 		let columnsResources: IDataResourceSchema[] = [];
 		columns.forEach(column => {
-			columnsResources.push({name: escape(column.columnName)});
+			columnsResources.push({ name: escape(column.columnName) });
 		});
 		let columnsFields: IDataResourceFields = { fields: undefined };
 		columnsFields.fields = columnsResources;
@@ -482,4 +524,17 @@ export interface IDataResourceFields {
 export interface IDataResourceSchema {
 	name: string;
 	type?: string;
-}
+}
+
+class ExternalScriptMagic {
+
+	constructor(private language: string) {
+	}
+
+	public convertToExternalScript(script: string): string {
+		return `execute sp_execute_external_script
+		@language = N'${this.language}',
+		@script = N'${script}'
+		`;
+	}
+}