Improve cell language detection and add support for language magics (#4081)

* Move to using notebook language by default, with override in cell
* Update cell language on kernel change
* Tweak language logic so that it prefers code mirror mode, then falls back since this was failing some notebooks
* Add new package.json contribution to define language magics. These result in cell language changing. Language is cleared out on removing the language magic
* Added support for executing Python, R and Java in the SQL Kernel to prove this out. It converts to the sp_execute_external_script format

TODO in future PR:

* Need to hook up completion item support for magics (issue #4078)
* Should add indicator at the bottom of a cell when an alternate language has been detected (issue #4079)
* On executing Python, R or Java, should add some output showing the generated code (issue #4080)

src/sql/workbench/services/notebook/common/notebookRegistry.ts

@@ -13,7 +13,8 @@ import * as sqlops from 'sqlops';
 import { Event, Emitter } from 'vs/base/common/event';
 
 export const Extensions = {
-	NotebookProviderContribution: 'notebook.providers'
+	NotebookProviderContribution: 'notebook.providers',
+	NotebookLanguageMagicContribution: 'notebook.languagemagics'
 };
 
 export interface NotebookProviderRegistration {
@@ -94,16 +95,67 @@ let notebookContrib: IJSONSchema = {
 		}
 	]
 };
+let notebookLanguageMagicType: IJSONSchema = {
+	type: 'object',
+	default: { magic: '', language: '', kernels: [], executionTarget: null },
+	properties: {
+		magic: {
+			description: localize('carbon.extension.contributes.notebook.magic', 'Name of the cell magic, such as "%%sql".'),
+			type: 'string'
+		},
+		language: {
+			description: localize('carbon.extension.contributes.notebook.language', 'The cell language to be used if this cell magic is included in the cell'),
+			type: 'string'
+		},
+		executionTarget: {
+			description: localize('carbon.extension.contributes.notebook.executionTarget', 'Optional execution target this magic indicates, for example Spark vs SQL'),
+			type: 'string'
+		},
+		kernels: {
+			description: localize('carbon.extension.contributes.notebook.kernels', 'Optional set of kernels this is valid for, e.g. python3, pyspark3, sql'),
+			oneOf: [
+				{ type: 'string' },
+				{
+					type: 'array',
+					items: {
+						type: 'string'
+					}
+				}
+			]
+		}
+	}
+};
+
+let languageMagicContrib: IJSONSchema = {
+	description: localize('vscode.extension.contributes.notebook.languagemagics', "Contributes notebook language."),
+	oneOf: [
+		notebookLanguageMagicType,
+		{
+			type: 'array',
+			items: notebookLanguageMagicType
+		}
+	]
+};
+
+export interface NotebookLanguageMagicRegistration {
+	magic: string;
+	language: string;
+	kernels?: string[];
+	executionTarget?: string;
+}
 
 export interface INotebookProviderRegistry {
-	readonly registrations: NotebookProviderRegistration[];
+	readonly providers: NotebookProviderRegistration[];
+	readonly languageMagics: NotebookLanguageMagicRegistration[];
 	readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }>;
 
-	registerNotebookProvider(registration: NotebookProviderRegistration): void;
+	registerNotebookProvider(provider: NotebookProviderRegistration): void;
+	registerNotebookLanguageMagic(magic: NotebookLanguageMagicRegistration): void;
 }
 
 class NotebookProviderRegistry implements INotebookProviderRegistry {
 	private providerIdToRegistration = new Map<string, NotebookProviderRegistration>();
+	private magicToRegistration = new Map<string, NotebookLanguageMagicRegistration>();
 	private _onNewRegistration = new Emitter<{ id: string, registration: NotebookProviderRegistration }>();
 	public readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }> = this._onNewRegistration.event;
 
@@ -114,11 +166,22 @@ class NotebookProviderRegistry implements INotebookProviderRegistry {
 		this._onNewRegistration.fire({ id: registration.provider, registration: registration });
 	}
 
-	public get registrations(): NotebookProviderRegistration[] {
+	public get providers(): NotebookProviderRegistration[] {
 		let registrationArray: NotebookProviderRegistration[] = [];
 		this.providerIdToRegistration.forEach(p => registrationArray.push(p));
 		return registrationArray;
 	}
+
+	registerNotebookLanguageMagic(magicRegistration: NotebookLanguageMagicRegistration): void {
+		this.magicToRegistration.set(magicRegistration.magic, magicRegistration);
+	}
+
+	public get languageMagics(): NotebookLanguageMagicRegistration[] {
+		let registrationArray: NotebookLanguageMagicRegistration[] = [];
+		this.magicToRegistration.forEach(p => registrationArray.push(p));
+		return registrationArray;
+	}
+
 }
 
 const notebookProviderRegistry = new NotebookProviderRegistry();
@@ -142,3 +205,21 @@ ExtensionsRegistry.registerExtensionPoint<NotebookProviderRegistration | Noteboo
 		}
 	}
 });
+
+ExtensionsRegistry.registerExtensionPoint<NotebookLanguageMagicRegistration | NotebookLanguageMagicRegistration[]>(Extensions.NotebookLanguageMagicContribution, [], languageMagicContrib).setHandler(extensions => {
+
+	function handleExtension(contrib: NotebookLanguageMagicRegistration, extension: IExtensionPointUser<any>) {
+		notebookProviderRegistry.registerNotebookLanguageMagic(contrib);
+	}
+
+	for (let extension of extensions) {
+		const { value } = extension;
+		if (Array.isArray<NotebookLanguageMagicRegistration>(value)) {
+			for (let command of value) {
+				handleExtension(command, extension);
+			}
+		} else {
+			handleExtension(value, extension);
+		}
+	}
+});

src/sql/workbench/services/notebook/common/notebookService.ts

@@ -16,7 +16,7 @@ import { ModelFactory } from 'sql/parts/notebook/models/modelFactory';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
 import { NotebookInput } from 'sql/parts/notebook/notebookInput';
 import { ISingleNotebookEditOperation } from 'sql/workbench/api/common/sqlExtHostTypes';
-import { ICellModel, INotebookModel } from 'sql/parts/notebook/models/modelInterfaces';
+import { ICellModel, INotebookModel, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 
 export const SERVICE_ID = 'notebookService';
 export const INotebookService = createDecorator<INotebookService>(SERVICE_ID);
@@ -35,6 +35,7 @@ export interface INotebookService {
 
 	readonly isRegistrationComplete: boolean;
 	readonly registrationComplete: Promise<void>;
+	readonly languageMagics: ILanguageMagic[];
 	/**
 	 * Register a metadata provider
 	 */

src/sql/workbench/services/notebook/common/notebookServiceImpl.ts

@@ -38,6 +38,7 @@ import { IEditorGroupsService } from 'vs/workbench/services/group/common/editorG
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IThemeService } from 'vs/platform/theme/common/themeService';
 import { registerNotebookThemes } from 'sql/parts/notebook/notebookStyles';
+import { ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 
 export interface NotebookProviderProperties {
 	provider: string;
@@ -333,6 +334,12 @@ export class NotebookService extends Disposable implements INotebookService {
 		}
 	}
 
+	get languageMagics(): ILanguageMagic[] {
+		return notebookRegistry.languageMagics;
+	}
+
+	// PRIVATE HELPERS /////////////////////////////////////////////////////
+
 	private sendNotebookCloseToProvider(editor: INotebookEditor): void {
 		let notebookUri = editor.notebookParams.notebookUri;
 		let uriString = notebookUri.toString();
@@ -347,7 +354,6 @@ export class NotebookService extends Disposable implements INotebookService {
 		}
 	}
 
-	// PRIVATE HELPERS /////////////////////////////////////////////////////
 	private async doWithProvider<T>(providerId: string, op: (provider: INotebookProvider) => Thenable<T>): Promise<T> {
 		// Make sure the provider exists before attempting to retrieve accounts
 		let provider: INotebookProvider = await this.getProviderInstance(providerId);
@@ -405,7 +411,7 @@ export class NotebookService extends Disposable implements INotebookService {
 	}
 
 	private cleanupProviders(): void {
-		let knownProviders = Object.keys(notebookRegistry.registrations);
+		let knownProviders = Object.keys(notebookRegistry.providers);
 		let cache = this.providersMemento.notebookProviderCache;
 		for (let key in cache) {
 			if (!knownProviders.includes(key)) {

src/sql/workbench/services/notebook/common/sqlSessionManager.ts

@@ -4,9 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+import * as os from 'os';
 import { nb, QueryExecuteSubsetResult, IDbColumn, BatchSummary, IResultMessage } from 'sqlops';
 import { localize } from 'vs/nls';
-import { FutureInternal } from 'sql/parts/notebook/models/modelInterfaces';
+import { FutureInternal, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces';
 import QueryRunner, { EventType } from 'sql/platform/query/common/queryRunner';
 import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@@ -18,6 +19,7 @@ import { IErrorMessageService } from 'sql/platform/errorMessage/common/errorMess
 import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile';
 import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
 import { escape } from 'sql/base/common/strings';
+import * as notebookUtils from 'sql/parts/notebook/notebookUtils';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 
 export const sqlKernel: string = localize('sqlKernel', 'SQL');
@@ -26,12 +28,23 @@ export const MAX_ROWS = 5000;
 export const NotebookConfigSectionName = 'notebook';
 export const MaxTableRowsConfigName = 'maxTableRows';
 
-let sqlKernelSpec: nb.IKernelSpec = ({
+const sqlKernelSpec: nb.IKernelSpec = ({
 	name: sqlKernel,
 	language: 'sql',
 	display_name: sqlKernel
 });
 
+const languageMagics: ILanguageMagic[] = [{
+	language: 'Python',
+	magic: 'lang_python'
+}, {
+	language: 'R',
+	magic: 'lang_r'
+}, {
+	language: 'Java',
+	magic: 'lang_java'
+}];
+
 export interface SQLData {
 	columns: Array<string>;
 	rows: Array<Array<string>>;
@@ -135,12 +148,22 @@ class SqlKernel extends Disposable implements nb.IKernel {
 	private _id: string;
 	private _future: SQLFuture;
 	private _executionCount: number = 0;
+	private _magicToExecutorMap = new Map<string, ExternalScriptMagic>();
 
-	constructor( @IConnectionManagementService private _connectionManagementService: IConnectionManagementService,
+	constructor(@IConnectionManagementService private _connectionManagementService: IConnectionManagementService,
 		@IInstantiationService private _instantiationService: IInstantiationService,
 		@IErrorMessageService private _errorMessageService: IErrorMessageService,
-		@IConfigurationService private _configurationService: IConfigurationService) {
+		@IConfigurationService private _configurationService: IConfigurationService
+	) {
 		super();
+		this.initMagics();
+	}
+
+	private initMagics(): void {
+		for (let magic of languageMagics) {
+			let scriptMagic = new ExternalScriptMagic(magic.language);
+			this._magicToExecutorMap.set(magic.magic, scriptMagic);
+		}
 	}
 
 	public get id(): string {
@@ -197,6 +220,7 @@ class SqlKernel extends Disposable implements nb.IKernel {
 
 	requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture {
 		let canRun: boolean = true;
+		let code = this.getCodeWithoutCellMagic(content);
 		if (this._queryRunner) {
 			// Cancel any existing query
 			if (this._future && !this._queryRunner.hasCompleted) {
@@ -204,14 +228,13 @@ class SqlKernel extends Disposable implements nb.IKernel {
 				// TODO when we can just show error as an output, should show an "execution canceled" error in output
 				this._future.handleDone();
 			}
-			this._queryRunner.runQuery(content.code);
+			this._queryRunner.runQuery(code);
 		} else if (this._currentConnection) {
 			let connectionUri = Utils.generateUri(this._currentConnection, 'notebook');
 			this._queryRunner = this._instantiationService.createInstance(QueryRunner, connectionUri);
-			this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) =>
-			{
+			this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) => {
 				this.addQueryEventListeners(this._queryRunner);
-				this._queryRunner.runQuery(content.code);
+				this._queryRunner.runQuery(code);
 			});
 		} else {
 			canRun = false;
@@ -231,6 +254,25 @@ class SqlKernel extends Disposable implements nb.IKernel {
 		return this._future;
 	}
 
+	private getCodeWithoutCellMagic(content: nb.IExecuteRequest): string {
+		let code = content.code;
+		let firstLineEnd = code.indexOf(os.EOL);
+		let firstLine = code.substring(0, (firstLineEnd >= 0) ? firstLineEnd : 0).trimLeft();
+		if (firstLine.startsWith('%%')) {
+			// Strip out the line
+			code = code.substring(firstLineEnd, code.length);
+			// Try and match to an external script magic. If we add more magics later, should handle transforms better
+			let magic = notebookUtils.tryMatchCellMagic(firstLine);
+			if (magic) {
+				let executor = this._magicToExecutorMap.get(magic.toLowerCase());
+				if (executor) {
+					code = executor.convertToExternalScript(code);
+				}
+			}
+		}
+		return code;
+	}
+
 	requestComplete(content: nb.ICompleteRequest): Thenable<nb.ICompleteReplyMsg> {
 		let response: Partial<nb.ICompleteReplyMsg> = {};
 		return Promise.resolve(response as nb.ICompleteReplyMsg);
@@ -388,7 +430,7 @@ export class SQLFuture extends Disposable implements FutureInternal {
 	private convertToDataResource(columns: IDbColumn[], subsetResult: QueryExecuteSubsetResult): IDataResource {
 		let columnsResources: IDataResourceSchema[] = [];
 		columns.forEach(column => {
-			columnsResources.push({name: escape(column.columnName)});
+			columnsResources.push({ name: escape(column.columnName) });
 		});
 		let columnsFields: IDataResourceFields = { fields: undefined };
 		columnsFields.fields = columnsResources;
@@ -482,4 +524,17 @@ export interface IDataResourceFields {
 export interface IDataResourceSchema {
 	name: string;
 	type?: string;
-}
+}
+
+class ExternalScriptMagic {
+
+	constructor(private language: string) {
+	}
+
+	public convertToExternalScript(script: string): string {
+		return `execute sp_execute_external_script
+		@language = N'${this.language}',
+		@script = N'${script}'
+		`;
+	}
+}