Improve cell language detection and add support for language magics (#4081)

* Move to using notebook language by default, with override in cell
* Update cell language on kernel change
* Tweak language logic so that it prefers code mirror mode, then falls back since this was failing some notebooks
* Add new package.json contribution to define language magics. These result in cell language changing. Language is cleared out on removing the language magic
* Added support for executing Python, R and Java in the SQL Kernel to prove this out. It converts to the sp_execute_external_script format

TODO in future PR:

* Need to hook up completion item support for magics (issue #4078)
* Should add indicator at the bottom of a cell when an alternate language has been detected (issue #4079)
* On executing Python, R or Java, should add some output showing the generated code (issue #4080)

src/sql/workbench/services/notebook/common/notebookRegistry.ts

@@ -13,7 +13,8 @@ import * as sqlops from 'sqlops';
 import { Event, Emitter } from 'vs/base/common/event';
 
 export const Extensions = {
-	NotebookProviderContribution: 'notebook.providers'
+	NotebookProviderContribution: 'notebook.providers',
+	NotebookLanguageMagicContribution: 'notebook.languagemagics'
 };
 
 export interface NotebookProviderRegistration {
@@ -94,16 +95,67 @@ let notebookContrib: IJSONSchema = {
 		}
 	]
 };
+let notebookLanguageMagicType: IJSONSchema = {
+	type: 'object',
+	default: { magic: '', language: '', kernels: [], executionTarget: null },
+	properties: {
+		magic: {
+			description: localize('carbon.extension.contributes.notebook.magic', 'Name of the cell magic, such as "%%sql".'),
+			type: 'string'
+		},
+		language: {
+			description: localize('carbon.extension.contributes.notebook.language', 'The cell language to be used if this cell magic is included in the cell'),
+			type: 'string'
+		},
+		executionTarget: {
+			description: localize('carbon.extension.contributes.notebook.executionTarget', 'Optional execution target this magic indicates, for example Spark vs SQL'),
+			type: 'string'
+		},
+		kernels: {
+			description: localize('carbon.extension.contributes.notebook.kernels', 'Optional set of kernels this is valid for, e.g. python3, pyspark3, sql'),
+			oneOf: [
+				{ type: 'string' },
+				{
+					type: 'array',
+					items: {
+						type: 'string'
+					}
+				}
+			]
+		}
+	}
+};
+
+let languageMagicContrib: IJSONSchema = {
+	description: localize('vscode.extension.contributes.notebook.languagemagics', "Contributes notebook language."),
+	oneOf: [
+		notebookLanguageMagicType,
+		{
+			type: 'array',
+			items: notebookLanguageMagicType
+		}
+	]
+};
+
+export interface NotebookLanguageMagicRegistration {
+	magic: string;
+	language: string;
+	kernels?: string[];
+	executionTarget?: string;
+}
 
 export interface INotebookProviderRegistry {
-	readonly registrations: NotebookProviderRegistration[];
+	readonly providers: NotebookProviderRegistration[];
+	readonly languageMagics: NotebookLanguageMagicRegistration[];
 	readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }>;
 
-	registerNotebookProvider(registration: NotebookProviderRegistration): void;
+	registerNotebookProvider(provider: NotebookProviderRegistration): void;
+	registerNotebookLanguageMagic(magic: NotebookLanguageMagicRegistration): void;
 }
 
 class NotebookProviderRegistry implements INotebookProviderRegistry {
 	private providerIdToRegistration = new Map<string, NotebookProviderRegistration>();
+	private magicToRegistration = new Map<string, NotebookLanguageMagicRegistration>();
 	private _onNewRegistration = new Emitter<{ id: string, registration: NotebookProviderRegistration }>();
 	public readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }> = this._onNewRegistration.event;
 
@@ -114,11 +166,22 @@ class NotebookProviderRegistry implements INotebookProviderRegistry {
 		this._onNewRegistration.fire({ id: registration.provider, registration: registration });
 	}
 
-	public get registrations(): NotebookProviderRegistration[] {
+	public get providers(): NotebookProviderRegistration[] {
 		let registrationArray: NotebookProviderRegistration[] = [];
 		this.providerIdToRegistration.forEach(p => registrationArray.push(p));
 		return registrationArray;
 	}
+
+	registerNotebookLanguageMagic(magicRegistration: NotebookLanguageMagicRegistration): void {
+		this.magicToRegistration.set(magicRegistration.magic, magicRegistration);
+	}
+
+	public get languageMagics(): NotebookLanguageMagicRegistration[] {
+		let registrationArray: NotebookLanguageMagicRegistration[] = [];
+		this.magicToRegistration.forEach(p => registrationArray.push(p));
+		return registrationArray;
+	}
+
 }
 
 const notebookProviderRegistry = new NotebookProviderRegistry();
@@ -142,3 +205,21 @@ ExtensionsRegistry.registerExtensionPoint<NotebookProviderRegistration | Noteboo
 		}
 	}
 });
+
+ExtensionsRegistry.registerExtensionPoint<NotebookLanguageMagicRegistration | NotebookLanguageMagicRegistration[]>(Extensions.NotebookLanguageMagicContribution, [], languageMagicContrib).setHandler(extensions => {
+
+	function handleExtension(contrib: NotebookLanguageMagicRegistration, extension: IExtensionPointUser<any>) {
+		notebookProviderRegistry.registerNotebookLanguageMagic(contrib);
+	}
+
+	for (let extension of extensions) {
+		const { value } = extension;
+		if (Array.isArray<NotebookLanguageMagicRegistration>(value)) {
+			for (let command of value) {
+				handleExtension(command, extension);
+			}
+		} else {
+			handleExtension(value, extension);
+		}
+	}
+});