diff --git a/extensions/notebook/package.json b/extensions/notebook/package.json index 12e6966c85..d1538e9bfe 100644 --- a/extensions/notebook/package.json +++ b/extensions/notebook/package.json @@ -130,6 +130,26 @@ "key": "Ctrl+Shift+T", "when": "notebookEditorVisible" } + ], + "notebook.languagemagics": [ + { + "magic": "lang_python", + "language": "python", + "executionTarget": null, + "kernels": ["sql"] + }, + { + "magic": "lang_r", + "language": "r", + "executionTarget": null, + "kernels": ["sql"] + }, + { + "magic": "lang_java", + "language": "java", + "executionTarget": null, + "kernels": ["sql"] + } ] }, "dependencies": { diff --git a/src/sql/parts/notebook/cellViews/code.component.ts b/src/sql/parts/notebook/cellViews/code.component.ts index f467af1c73..6b8bd4bff7 100644 --- a/src/sql/parts/notebook/cellViews/code.component.ts +++ b/src/sql/parts/notebook/cellViews/code.component.ts @@ -6,7 +6,6 @@ import 'vs/css!./code'; import { OnInit, Component, Input, Inject, forwardRef, ElementRef, ChangeDetectorRef, ViewChild, Output, EventEmitter, OnChanges, SimpleChange } from '@angular/core'; -import { CommonServiceInterface } from 'sql/services/common/commonServiceInterface.service'; import { AngularDisposable } from 'sql/base/node/lifecycle'; import { QueryTextEditor } from 'sql/parts/modelComponents/queryTextEditor'; import { CellToggleMoreActions } from 'sql/parts/notebook/cellToggleMoreActions'; @@ -26,12 +25,12 @@ import { UntitledEditorInput } from 'vs/workbench/common/editor/untitledEditorIn import * as DOM from 'vs/base/browser/dom'; import { IModeService } from 'vs/editor/common/services/modeService'; import { IModelService } from 'vs/editor/common/services/modelService'; -import { IContextMenuService, IContextViewService } from 'vs/platform/contextview/browser/contextView'; -import { INotificationService } from 'vs/platform/notification/common/notification'; +import { IContextMenuService } from 'vs/platform/contextview/browser/contextView'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { Emitter, debounceEvent } from 'vs/base/common/event'; import { CellTypes } from 'sql/parts/notebook/models/contracts'; import { OVERRIDE_EDITOR_THEMING_SETTING } from 'sql/workbench/services/notebook/common/notebookService'; +import * as notebookUtils from 'sql/parts/notebook/notebookUtils'; export const CODE_SELECTOR: string = 'code-component'; const MARKDOWN_CLASS = 'markdown'; @@ -61,6 +60,12 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange @Input() set model(value: NotebookModel) { this._model = value; + this._register(value.kernelChanged(() => { + // On kernel change, need to reevaluate the language for each cell + // Refresh based on the cell magic (since this is kernel-dependent) and then update using notebook language + this.checkForLanguageMagics(); + this.updateLanguageMode(); + })); } @Input() set activeCellId(value: string) { @@ -88,21 +93,17 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange private _layoutEmitter = new Emitter(); constructor( - @Inject(forwardRef(() => CommonServiceInterface)) private _bootstrapService: CommonServiceInterface, - @Inject(forwardRef(() => ChangeDetectorRef)) private _changeRef: ChangeDetectorRef, @Inject(IWorkbenchThemeService) private themeService: IWorkbenchThemeService, @Inject(IInstantiationService) private _instantiationService: IInstantiationService, @Inject(IModelService) private _modelService: IModelService, @Inject(IModeService) private _modeService: IModeService, @Inject(IContextMenuService) private contextMenuService: IContextMenuService, - @Inject(IContextViewService) private contextViewService: IContextViewService, - @Inject(INotificationService) private notificationService: INotificationService, @Inject(IConfigurationService) private _configurationService: IConfigurationService ) { super(); this._cellToggleMoreActions = this._instantiationService.createInstance(CellToggleMoreActions); - debounceEvent(this._layoutEmitter.event, (l, e) => e, 250, /*leading=*/false) - (() => this.layout()); + this._register(debounceEvent(this._layoutEmitter.event, (l, e) => e, 250, /*leading=*/false) + (() => this.layout())); } @@ -180,6 +181,7 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange this._editor.setHeightToScrollHeight(); this.cellModel.source = this._editorModel.getValue(); this.onContentChanged.emit(); + this.checkForLanguageMagics(); // TODO see if there's a better way to handle reassessing size. setTimeout(() => this._layoutEmitter.fire(), 250); })); @@ -220,7 +222,31 @@ export class CodeComponent extends AngularDisposable implements OnInit, OnChange } } - private updateLanguageMode() { + private checkForLanguageMagics(): void { + try { + if (!this.cellModel || this.cellModel.cellType !== CellTypes.Code) { + return; + } + if (this._editorModel && this._editor && this._editorModel.getLineCount() > 1) { + // Only try to match once we've typed past the first line + let magicName = notebookUtils.tryMatchCellMagic(this._editorModel.getLineContent(1)); + if (magicName) { + let kernelName = this._model.clientSession && this._model.clientSession.kernel ? this._model.clientSession.kernel.name : undefined; + let magic = this._model.notebookOptions.cellMagicMapper.toLanguageMagic(magicName, kernelName); + if (magic && this.cellModel.language !== magic.language) { + this.cellModel.setOverrideLanguage(magic.language); + this.updateLanguageMode(); + } + } else { + this.cellModel.setOverrideLanguage(undefined); + } + } + } catch (err) { + // No-op for now. Should we log? + } + } + + private updateLanguageMode(): void { if (this._editorModel && this._editor) { this._modeService.getOrCreateMode(this.cellModel.language).then((modeValue) => { this._modelService.setMode(this._editorModel, modeValue); diff --git a/src/sql/parts/notebook/models/cell.ts b/src/sql/parts/notebook/models/cell.ts index 9e8601e9f9..267ac57993 100644 --- a/src/sql/parts/notebook/models/cell.ts +++ b/src/sql/parts/notebook/models/cell.ts @@ -49,7 +49,6 @@ export class CellModel implements ICellModel { this._source = ''; } this._isEditMode = this._cellType !== CellTypes.Markdown; - this.ensureDefaultLanguage(); if (_options && _options.isTrusted) { this._isTrusted = true; } else { @@ -150,10 +149,16 @@ export class CellModel implements ICellModel { } public get language(): string { - return this._language; + if (this._cellType === CellTypes.Markdown) { + return 'markdown'; + } + if (this._language) { + return this._language; + } + return this.options.notebook.language; } - public set language(newLanguage: string) { + public setOverrideLanguage(newLanguage: string) { this._language = newLanguage; } @@ -203,7 +208,7 @@ export class CellModel implements ICellModel { }, false); this.setFuture(future as FutureInternal); // For now, await future completion. Later we should just track and handle cancellation based on model notifications - let result: nb.IExecuteReplyMsg = await future.done; + let result: nb.IExecuteReplyMsg = await future.done; if (result && result.content) { this.executionCount = result.content.execution_count; if (result.content.status !== 'ok') { @@ -254,7 +259,7 @@ export class CellModel implements ICellModel { private sendNotification(notificationService: INotificationService, severity: Severity, message: string): void { if (notificationService) { - notificationService.notify({ severity: severity, message: message}); + notificationService.notify({ severity: severity, message: message }); } } @@ -382,7 +387,7 @@ export class CellModel implements ICellModel { } } } - catch (e) {} + catch (e) { } } return output; } @@ -401,7 +406,7 @@ export class CellModel implements ICellModel { }; if (this._cellType === CellTypes.Code) { cellJson.metadata.language = this._language, - cellJson.outputs = this._outputs; + cellJson.outputs = this._outputs; cellJson.execution_count = this.executionCount; } return cellJson as nb.ICellContents; @@ -437,77 +442,15 @@ export class CellModel implements ICellModel { this._outputs.push(output); } - /** - * Normalize an output. - */ - private _normalize(value: nb.ICellOutput): void { - if (notebookUtils.isStream(value)) { - if (Array.isArray(value.text)) { - value.text = (value.text as string[]).join('\n'); - } - } - } - - private get languageInfo(): nb.ILanguageInfo { - if (this._options && this._options.notebook && this._options.notebook.languageInfo) { - return this._options.notebook.languageInfo; - } - return undefined; - } - /** - * Ensures there is a default language set, if none was already defined. - * Will read information from the overall Notebook (passed as options to the model), or - * if all else fails default back to python. - * + * Normalize an output. */ - private ensureDefaultLanguage(): void { - // See if language is already set / is known based on cell type - if (this.hasLanguage()) { - return; - } - if (this._cellType === CellTypes.Markdown) { - this._language = 'markdown'; - return; - } - - // try set it based on overall Notebook language - this.trySetLanguageFromLangInfo(); - - // fallback to python - if (!this._language) { - this._language = 'python'; - } - } - - private trySetLanguageFromLangInfo() { - // In languageInfo, set the language to the "name" property - // If the "name" property isn't defined, check the "mimeType" property - // Otherwise, default to python as the language - let languageInfo = this.languageInfo; - if (languageInfo) { - if (languageInfo.name) { - this._language = languageInfo.name; - } else if (languageInfo.codemirror_mode) { - let codeMirrorMode: nb.ICodeMirrorMode = (languageInfo.codemirror_mode); - if (codeMirrorMode && codeMirrorMode.name) { - this._language = codeMirrorMode.name; - } - } else if (languageInfo.mimetype) { - this._language = languageInfo.mimetype; + private _normalize(value: nb.ICellOutput): void { + if (notebookUtils.isStream(value)) { + if (Array.isArray(value.text)) { + value.text = (value.text as string[]).join('\n'); } } - - if (this._language) { - let mimeTypePrefix = 'x-'; - if (this._language.includes(mimeTypePrefix)) { - this._language = this._language.replace(mimeTypePrefix, ''); - } - } - } - - private hasLanguage(): boolean { - return !!this._language; } private createUri(): void { diff --git a/src/sql/parts/notebook/models/cellMagicMapper.ts b/src/sql/parts/notebook/models/cellMagicMapper.ts new file mode 100644 index 0000000000..1cd2fcb0f4 --- /dev/null +++ b/src/sql/parts/notebook/models/cellMagicMapper.ts @@ -0,0 +1,55 @@ + +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Microsoft Corporation. All rights reserved. +* Licensed under the Source EULA. See License.txt in the project root for license information. +*--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { ICellMagicMapper, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces'; + +const defaultKernel = '*'; +export class CellMagicMapper implements ICellMagicMapper { + private kernelToMagicMap = new Map(); + + constructor(languageMagics: ILanguageMagic[]) { + if (languageMagics) { + for (let magic of languageMagics) { + if (!magic.kernels || magic.kernels.length === 0) { + this.addKernelMapping(defaultKernel, magic); + } + if (magic.kernels) { + for (let kernel of magic.kernels) { + this.addKernelMapping(kernel.toLowerCase(), magic); + } + } + } + } + } + + private addKernelMapping(kernelId: string, magic: ILanguageMagic): void { + let magics = this.kernelToMagicMap.get(kernelId) || []; + magics.push(magic); + this.kernelToMagicMap.set(kernelId, magics); + } + + private findMagicForKernel(searchText: string, kernelId: string): ILanguageMagic | undefined { + if (kernelId === undefined || !searchText) { + return undefined; + } + searchText = searchText.toLowerCase(); + let kernelMagics = this.kernelToMagicMap.get(kernelId) || []; + if (kernelMagics) { + return kernelMagics.find(m => m.magic.toLowerCase() === searchText); + } + return undefined; + } + + toLanguageMagic(magic: string, kernelId: string): ILanguageMagic { + let languageMagic = this.findMagicForKernel(magic, kernelId.toLowerCase()); + if (!languageMagic) { + languageMagic = this.findMagicForKernel(magic, defaultKernel); + } + return languageMagic; + } +} diff --git a/src/sql/parts/notebook/models/modelInterfaces.ts b/src/sql/parts/notebook/models/modelInterfaces.ts index b09b794210..1ddde7d1b7 100644 --- a/src/sql/parts/notebook/models/modelInterfaces.ts +++ b/src/sql/parts/notebook/models/modelInterfaces.ts @@ -267,9 +267,13 @@ export interface INotebookModel { */ readonly clientSession: IClientSession; /** - * LanguageInfo saved in the query book + * LanguageInfo saved in the notebook */ readonly languageInfo: nb.ILanguageInfo; + /** + * Current default language for the notebook + */ + readonly language: string; /** * All notebook managers applicable for a given notebook @@ -421,7 +425,7 @@ export enum CellExecutionState { export interface ICellModel { cellUri: URI; id: string; - language: string; + readonly language: string; source: string; cellType: CellType; trustedMode: boolean; @@ -435,6 +439,7 @@ export interface ICellModel { setFuture(future: FutureInternal): void; readonly executionState: CellExecutionState; runCell(notificationService?: INotificationService): Promise; + setOverrideLanguage(language: string); equals(cellModel: ICellModel): boolean; toJSON(): nb.ICellContents; } @@ -465,6 +470,7 @@ export interface INotebookModelOptions { providerId: string; standardKernels: IStandardKernelWithProvider[]; defaultKernel: nb.IKernelSpec; + cellMagicMapper: ICellMagicMapper; layoutChanged: Event; @@ -473,6 +479,22 @@ export interface INotebookModelOptions { capabilitiesService: ICapabilitiesService; } +export interface ILanguageMagic { + magic: string; + language: string; + kernels?: string[]; + executionTarget?: string; +} + +export interface ICellMagicMapper { + /** + * Tries to find a language mapping for an identified cell magic + * @param magic a string defining magic. For example for %%sql the magic text is sql + * @param kernelId the name of the current kernel to use when looking up magics + */ + toLanguageMagic(magic: string, kernelId: string): ILanguageMagic | undefined; +} + export namespace notebookConstants { export const SQL = 'SQL'; } \ No newline at end of file diff --git a/src/sql/parts/notebook/models/notebookContexts.ts b/src/sql/parts/notebook/models/notebookContexts.ts index 29a144544f..2c8b87e459 100644 --- a/src/sql/parts/notebook/models/notebookContexts.ts +++ b/src/sql/parts/notebook/models/notebookContexts.ts @@ -8,7 +8,7 @@ import { nb } from 'sqlops'; import { localize } from 'vs/nls'; -import { IDefaultConnection, notebookConstants, INotebookModelOptions } from 'sql/parts/notebook/models/modelInterfaces'; +import { IDefaultConnection, notebookConstants } from 'sql/parts/notebook/models/modelInterfaces'; import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement'; import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile'; import { IConnectionProfile } from 'sql/platform/connection/common/interfaces'; diff --git a/src/sql/parts/notebook/models/notebookModel.ts b/src/sql/parts/notebook/models/notebookModel.ts index 714b5e9ae3..bb17fbca8d 100644 --- a/src/sql/parts/notebook/models/notebookModel.ts +++ b/src/sql/parts/notebook/models/notebookModel.ts @@ -56,6 +56,7 @@ export class NotebookModel extends Disposable implements INotebookModel { private _cells: ICellModel[]; private _defaultLanguageInfo: nb.ILanguageInfo; + private _language: string; private _onErrorEmitter = new Emitter(); private _savedKernelInfo: nb.IKernelInfo; private readonly _nbformat: number = nbversion.MAJOR_VERSION; @@ -68,31 +69,31 @@ export class NotebookModel extends Disposable implements INotebookModel { private _kernelDisplayNameToNotebookProviderIds: Map = new Map(); private _onValidConnectionSelected = new Emitter(); - constructor(public notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) { + constructor(private _notebookOptions: INotebookModelOptions, startSessionImmediately?: boolean, private connectionProfile?: IConnectionProfile) { super(); - if (!notebookOptions || !notebookOptions.notebookUri || !notebookOptions.notebookManagers) { + if (!_notebookOptions || !_notebookOptions.notebookUri || !_notebookOptions.notebookManagers) { throw new Error('path or notebook service not defined'); } if (startSessionImmediately) { this.backgroundStartSession(); } this._trustedMode = false; - this._providerId = notebookOptions.providerId; + this._providerId = _notebookOptions.providerId; this._onProviderIdChanged.fire(this._providerId); - this.notebookOptions.standardKernels.forEach(kernel => { + this._notebookOptions.standardKernels.forEach(kernel => { this._kernelDisplayNameToConnectionProviderIds.set(kernel.name, kernel.connectionProviderIds); this._kernelDisplayNameToNotebookProviderIds.set(kernel.name, kernel.notebookProvider); }); - if (this.notebookOptions.layoutChanged) { - this.notebookOptions.layoutChanged(() => this._layoutChanged.fire()); + if (this._notebookOptions.layoutChanged) { + this._notebookOptions.layoutChanged(() => this._layoutChanged.fire()); } - this._defaultKernel = notebookOptions.defaultKernel; + this._defaultKernel = _notebookOptions.defaultKernel; } public get notebookManagers(): INotebookManager[] { - let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER); + let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER); if (!notebookManagers.length) { - return this.notebookOptions.notebookManagers; + return this._notebookOptions.notebookManagers; } return notebookManagers; } @@ -107,11 +108,15 @@ export class NotebookModel extends Disposable implements INotebookModel { return manager; } + public get notebookOptions(): INotebookModelOptions { + return this._notebookOptions; + } + public get notebookUri(): URI { - return this.notebookOptions.notebookUri; + return this._notebookOptions.notebookUri; } public set notebookUri(value: URI) { - this.notebookOptions.notebookUri = value; + this._notebookOptions.notebookUri = value; } public get hasServerManager(): boolean { @@ -246,11 +251,11 @@ export class NotebookModel extends Disposable implements INotebookModel { try { this._trustedMode = isTrusted; let contents = null; - if (this.notebookOptions.notebookUri.scheme !== Schemas.untitled) { + if (this._notebookOptions.notebookUri.scheme !== Schemas.untitled) { // TODO: separate ContentManager from NotebookManager - contents = await this.notebookManagers[0].contentManager.getNotebookContents(this.notebookOptions.notebookUri); + contents = await this.notebookManagers[0].contentManager.getNotebookContents(this._notebookOptions.notebookUri); } - let factory = this.notebookOptions.factory; + let factory = this._notebookOptions.factory; // if cells already exist, create them with language info (if it is saved) this._cells = []; this._defaultLanguageInfo = { @@ -268,6 +273,7 @@ export class NotebookModel extends Disposable implements INotebookModel { this._cells = contents.cells.map(c => factory.createCell(c, { notebook: this, isTrusted: isTrusted })); } } + this.trySetLanguageFromLangInfo(); } catch (error) { this._inErrorState = true; throw error; @@ -317,7 +323,7 @@ export class NotebookModel extends Disposable implements INotebookModel { metadata: {}, execution_count: undefined }; - return this.notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true }); + return this._notebookOptions.factory.createCell(singleCell, { notebook: this, isTrusted: true }); } deleteCell(cellModel: ICellModel): void { @@ -347,7 +353,7 @@ export class NotebookModel extends Disposable implements INotebookModel { if (edit.cell) { // TODO: should we validate and complete required missing parameters? let contents: nb.ICellContents = edit.cell as nb.ICellContents; - newCells.push(this.notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode })); + newCells.push(this._notebookOptions.factory.createCell(contents, { notebook: this, isTrusted: this._trustedMode })); } this._cells.splice(edit.range.start, edit.range.end - edit.range.start, ...newCells); if (newCells.length > 0) { @@ -374,16 +380,16 @@ export class NotebookModel extends Disposable implements INotebookModel { public backgroundStartSession(): void { // TODO: only one session should be active at a time, depending on the current provider this.notebookManagers.forEach(manager => { - let clientSession = this.notebookOptions.factory.createClientSession({ - notebookUri: this.notebookOptions.notebookUri, + let clientSession = this._notebookOptions.factory.createClientSession({ + notebookUri: this._notebookOptions.notebookUri, notebookManager: manager, - notificationService: this.notebookOptions.notificationService + notificationService: this._notebookOptions.notificationService }); this._clientSessions.push(clientSession); if (!this._activeClientSession) { this._activeClientSession = clientSession; } - let profile = new ConnectionProfile(this.notebookOptions.capabilitiesService, this.connectionProfile); + let profile = new ConnectionProfile(this._notebookOptions.capabilitiesService, this.connectionProfile); if (this.isValidConnection(profile)) { this._activeConnection = profile; @@ -405,7 +411,7 @@ export class NotebookModel extends Disposable implements INotebookModel { } private isValidConnection(profile: IConnectionProfile | connection.Connection) { - let standardKernels = this.notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name); + let standardKernels = this._notebookOptions.standardKernels.find(kernel => this._savedKernelInfo && kernel.name === this._savedKernelInfo.display_name); let connectionProviderIds = standardKernels ? standardKernels.connectionProviderIds : undefined; return profile && connectionProviderIds && connectionProviderIds.find(provider => provider === profile.providerName) !== undefined; } @@ -414,12 +420,51 @@ export class NotebookModel extends Disposable implements INotebookModel { return this._defaultLanguageInfo; } + public get language(): string { + return this._language; + } + private updateLanguageInfo(info: nb.ILanguageInfo) { if (info) { this._defaultLanguageInfo = info; + this.trySetLanguageFromLangInfo(); } } + private trySetLanguageFromLangInfo() { + // In languageInfo, set the language to the "name" property + // If the "name" property isn't defined, check the "mimeType" property + // Otherwise, default to python as the language + let languageInfo = this.languageInfo; + let language: string; + if (languageInfo) { + if (languageInfo.codemirror_mode) { + let codeMirrorMode: nb.ICodeMirrorMode = (languageInfo.codemirror_mode); + if (codeMirrorMode && codeMirrorMode.name) { + language = codeMirrorMode.name; + } + } + if (!language && languageInfo.name) { + language = languageInfo.name; + } + if (!language && languageInfo.mimetype) { + language = languageInfo.mimetype; + } + } + + if (language) { + let mimeTypePrefix = 'x-'; + if (language.includes(mimeTypePrefix)) { + language = language.replace(mimeTypePrefix, ''); + } else if (language.toLowerCase() === 'ipython') { + // Special case ipython because in many cases this is defined as the code mirror mode for python notebooks + language = 'python'; + } + } + + this._language = language; + } + public changeKernel(displayName: string): void { let spec = this.getKernelSpecFromDisplayName(displayName); this.doChangeKernel(spec); @@ -457,7 +502,7 @@ export class NotebookModel extends Disposable implements INotebookModel { if (!newConnection && (this._activeContexts.defaultConnection.serverName === server)) { newConnection = this._activeContexts.defaultConnection; } - let newConnectionProfile = new ConnectionProfile(this.notebookOptions.capabilitiesService, newConnection); + let newConnectionProfile = new ConnectionProfile(this._notebookOptions.capabilitiesService, newConnection); this._activeConnection = newConnectionProfile; this.refreshConnections(newConnectionProfile); this._activeClientSession.updateConnection(this._activeConnection.toIConnectionProfile()).then( @@ -593,7 +638,7 @@ export class NotebookModel extends Disposable implements INotebookModel { private async loadActiveContexts(kernelChangedArgs: nb.IKernelChangedArgs): Promise { if (kernelChangedArgs && kernelChangedArgs.newValue && kernelChangedArgs.newValue.name) { let kernelDisplayName = this.getDisplayNameFromSpecName(kernelChangedArgs.newValue); - this._activeContexts = await NotebookContexts.getContextsForKernel(this.notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile); + this._activeContexts = await NotebookContexts.getContextsForKernel(this._notebookOptions.connectionService, this.getApplicableConnectionProviderIds(kernelDisplayName), kernelChangedArgs, this.connectionProfile); this._contextsChangedEmitter.fire(); if (this.contexts.defaultConnection !== undefined && this.contexts.defaultConnection.serverName !== undefined) { await this.changeContext(this.contexts.defaultConnection.serverName); @@ -622,7 +667,7 @@ export class NotebookModel extends Disposable implements INotebookModel { return false; } // TODO: refactor ContentManager out from NotebookManager - await this.notebookManagers[0].contentManager.save(this.notebookOptions.notebookUri, notebook); + await this.notebookManagers[0].contentManager.save(this._notebookOptions.notebookUri, notebook); this._contentChangedEmitter.fire({ changeType: NotebookChangeType.DirtyStateChanged, isDirty: false @@ -653,9 +698,9 @@ export class NotebookModel extends Disposable implements INotebookModel { private setProviderIdForKernel(kernelSpec: nb.IKernelSpec): void { if (!kernelSpec) { // Just use the 1st non-default provider, we don't have a better heuristic - let notebookManagers = this.notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER); + let notebookManagers = this._notebookOptions.notebookManagers.filter(manager => manager.providerId !== DEFAULT_NOTEBOOK_PROVIDER); if (!notebookManagers.length) { - notebookManagers = this.notebookOptions.notebookManagers; + notebookManagers = this._notebookOptions.notebookManagers; } if (notebookManagers.length > 0) { this._providerId = notebookManagers[0].providerId; diff --git a/src/sql/parts/notebook/notebook.component.ts b/src/sql/parts/notebook/notebook.component.ts index cf172672e6..19a0636ef0 100644 --- a/src/sql/parts/notebook/notebook.component.ts +++ b/src/sql/parts/notebook/notebook.component.ts @@ -49,6 +49,7 @@ import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/un import { IEditorGroupsService } from 'vs/workbench/services/group/common/editorGroupsService'; import { IConnectionDialogService } from 'sql/workbench/services/connection/common/connectionDialogService'; import { ICapabilitiesService } from 'sql/platform/capabilities/common/capabilitiesService'; +import { CellMagicMapper } from 'sql/parts/notebook/models/cellMagicMapper'; export const NOTEBOOK_SELECTOR: string = 'notebook-component'; @@ -263,6 +264,7 @@ export class NotebookComponent extends AngularDisposable implements OnInit, OnDe notificationService: this.notificationService, notebookManagers: this.notebookManagers, standardKernels: this._notebookParams.input.standardKernels, + cellMagicMapper: new CellMagicMapper(this.notebookService.languageMagics), providerId: notebookUtils.sqlNotebooksEnabled(this.contextKeyService) ? 'sql' : 'jupyter', // this is tricky; really should also depend on the connection profile defaultKernel: this._notebookParams.input.defaultKernel, layoutChanged: this._notebookParams.input.layoutChanged, diff --git a/src/sql/parts/notebook/notebookUtils.ts b/src/sql/parts/notebook/notebookUtils.ts index 5c15fb52fb..cbc8be4959 100644 --- a/src/sql/parts/notebook/notebookUtils.ts +++ b/src/sql/parts/notebook/notebookUtils.ts @@ -104,4 +104,15 @@ export interface IStandardKernelWithProvider { readonly name: string; readonly connectionProviderIds: string[]; readonly notebookProvider: string; +} + +export function tryMatchCellMagic(input: string): string { + if (!input) { + return input; + } + let firstLine = input.trimLeft(); + let magicRegex = /^%%(\w+)/g; + let match = magicRegex.exec(firstLine); + let magicName = match && match[1]; + return magicName; } \ No newline at end of file diff --git a/src/sql/workbench/services/notebook/common/notebookRegistry.ts b/src/sql/workbench/services/notebook/common/notebookRegistry.ts index 0bb4a1add4..61934dee50 100644 --- a/src/sql/workbench/services/notebook/common/notebookRegistry.ts +++ b/src/sql/workbench/services/notebook/common/notebookRegistry.ts @@ -13,7 +13,8 @@ import * as sqlops from 'sqlops'; import { Event, Emitter } from 'vs/base/common/event'; export const Extensions = { - NotebookProviderContribution: 'notebook.providers' + NotebookProviderContribution: 'notebook.providers', + NotebookLanguageMagicContribution: 'notebook.languagemagics' }; export interface NotebookProviderRegistration { @@ -94,16 +95,67 @@ let notebookContrib: IJSONSchema = { } ] }; +let notebookLanguageMagicType: IJSONSchema = { + type: 'object', + default: { magic: '', language: '', kernels: [], executionTarget: null }, + properties: { + magic: { + description: localize('carbon.extension.contributes.notebook.magic', 'Name of the cell magic, such as "%%sql".'), + type: 'string' + }, + language: { + description: localize('carbon.extension.contributes.notebook.language', 'The cell language to be used if this cell magic is included in the cell'), + type: 'string' + }, + executionTarget: { + description: localize('carbon.extension.contributes.notebook.executionTarget', 'Optional execution target this magic indicates, for example Spark vs SQL'), + type: 'string' + }, + kernels: { + description: localize('carbon.extension.contributes.notebook.kernels', 'Optional set of kernels this is valid for, e.g. python3, pyspark3, sql'), + oneOf: [ + { type: 'string' }, + { + type: 'array', + items: { + type: 'string' + } + } + ] + } + } +}; + +let languageMagicContrib: IJSONSchema = { + description: localize('vscode.extension.contributes.notebook.languagemagics', "Contributes notebook language."), + oneOf: [ + notebookLanguageMagicType, + { + type: 'array', + items: notebookLanguageMagicType + } + ] +}; + +export interface NotebookLanguageMagicRegistration { + magic: string; + language: string; + kernels?: string[]; + executionTarget?: string; +} export interface INotebookProviderRegistry { - readonly registrations: NotebookProviderRegistration[]; + readonly providers: NotebookProviderRegistration[]; + readonly languageMagics: NotebookLanguageMagicRegistration[]; readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }>; - registerNotebookProvider(registration: NotebookProviderRegistration): void; + registerNotebookProvider(provider: NotebookProviderRegistration): void; + registerNotebookLanguageMagic(magic: NotebookLanguageMagicRegistration): void; } class NotebookProviderRegistry implements INotebookProviderRegistry { private providerIdToRegistration = new Map(); + private magicToRegistration = new Map(); private _onNewRegistration = new Emitter<{ id: string, registration: NotebookProviderRegistration }>(); public readonly onNewRegistration: Event<{ id: string, registration: NotebookProviderRegistration }> = this._onNewRegistration.event; @@ -114,11 +166,22 @@ class NotebookProviderRegistry implements INotebookProviderRegistry { this._onNewRegistration.fire({ id: registration.provider, registration: registration }); } - public get registrations(): NotebookProviderRegistration[] { + public get providers(): NotebookProviderRegistration[] { let registrationArray: NotebookProviderRegistration[] = []; this.providerIdToRegistration.forEach(p => registrationArray.push(p)); return registrationArray; } + + registerNotebookLanguageMagic(magicRegistration: NotebookLanguageMagicRegistration): void { + this.magicToRegistration.set(magicRegistration.magic, magicRegistration); + } + + public get languageMagics(): NotebookLanguageMagicRegistration[] { + let registrationArray: NotebookLanguageMagicRegistration[] = []; + this.magicToRegistration.forEach(p => registrationArray.push(p)); + return registrationArray; + } + } const notebookProviderRegistry = new NotebookProviderRegistry(); @@ -142,3 +205,21 @@ ExtensionsRegistry.registerExtensionPoint(Extensions.NotebookLanguageMagicContribution, [], languageMagicContrib).setHandler(extensions => { + + function handleExtension(contrib: NotebookLanguageMagicRegistration, extension: IExtensionPointUser) { + notebookProviderRegistry.registerNotebookLanguageMagic(contrib); + } + + for (let extension of extensions) { + const { value } = extension; + if (Array.isArray(value)) { + for (let command of value) { + handleExtension(command, extension); + } + } else { + handleExtension(value, extension); + } + } +}); diff --git a/src/sql/workbench/services/notebook/common/notebookService.ts b/src/sql/workbench/services/notebook/common/notebookService.ts index 09d6832d22..1afb4b1f51 100644 --- a/src/sql/workbench/services/notebook/common/notebookService.ts +++ b/src/sql/workbench/services/notebook/common/notebookService.ts @@ -16,7 +16,7 @@ import { ModelFactory } from 'sql/parts/notebook/models/modelFactory'; import { IConnectionProfile } from 'sql/platform/connection/common/interfaces'; import { NotebookInput } from 'sql/parts/notebook/notebookInput'; import { ISingleNotebookEditOperation } from 'sql/workbench/api/common/sqlExtHostTypes'; -import { ICellModel, INotebookModel } from 'sql/parts/notebook/models/modelInterfaces'; +import { ICellModel, INotebookModel, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces'; export const SERVICE_ID = 'notebookService'; export const INotebookService = createDecorator(SERVICE_ID); @@ -35,6 +35,7 @@ export interface INotebookService { readonly isRegistrationComplete: boolean; readonly registrationComplete: Promise; + readonly languageMagics: ILanguageMagic[]; /** * Register a metadata provider */ diff --git a/src/sql/workbench/services/notebook/common/notebookServiceImpl.ts b/src/sql/workbench/services/notebook/common/notebookServiceImpl.ts index b84fe01e91..029db808bf 100644 --- a/src/sql/workbench/services/notebook/common/notebookServiceImpl.ts +++ b/src/sql/workbench/services/notebook/common/notebookServiceImpl.ts @@ -38,6 +38,7 @@ import { IEditorGroupsService } from 'vs/workbench/services/group/common/editorG import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { IThemeService } from 'vs/platform/theme/common/themeService'; import { registerNotebookThemes } from 'sql/parts/notebook/notebookStyles'; +import { ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces'; export interface NotebookProviderProperties { provider: string; @@ -333,6 +334,12 @@ export class NotebookService extends Disposable implements INotebookService { } } + get languageMagics(): ILanguageMagic[] { + return notebookRegistry.languageMagics; + } + + // PRIVATE HELPERS ///////////////////////////////////////////////////// + private sendNotebookCloseToProvider(editor: INotebookEditor): void { let notebookUri = editor.notebookParams.notebookUri; let uriString = notebookUri.toString(); @@ -347,7 +354,6 @@ export class NotebookService extends Disposable implements INotebookService { } } - // PRIVATE HELPERS ///////////////////////////////////////////////////// private async doWithProvider(providerId: string, op: (provider: INotebookProvider) => Thenable): Promise { // Make sure the provider exists before attempting to retrieve accounts let provider: INotebookProvider = await this.getProviderInstance(providerId); @@ -405,7 +411,7 @@ export class NotebookService extends Disposable implements INotebookService { } private cleanupProviders(): void { - let knownProviders = Object.keys(notebookRegistry.registrations); + let knownProviders = Object.keys(notebookRegistry.providers); let cache = this.providersMemento.notebookProviderCache; for (let key in cache) { if (!knownProviders.includes(key)) { diff --git a/src/sql/workbench/services/notebook/common/sqlSessionManager.ts b/src/sql/workbench/services/notebook/common/sqlSessionManager.ts index 6d16fa5f83..31df548932 100644 --- a/src/sql/workbench/services/notebook/common/sqlSessionManager.ts +++ b/src/sql/workbench/services/notebook/common/sqlSessionManager.ts @@ -4,9 +4,10 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; +import * as os from 'os'; import { nb, QueryExecuteSubsetResult, IDbColumn, BatchSummary, IResultMessage } from 'sqlops'; import { localize } from 'vs/nls'; -import { FutureInternal } from 'sql/parts/notebook/models/modelInterfaces'; +import { FutureInternal, ILanguageMagic } from 'sql/parts/notebook/models/modelInterfaces'; import QueryRunner, { EventType } from 'sql/platform/query/common/queryRunner'; import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; @@ -18,6 +19,7 @@ import { IErrorMessageService } from 'sql/platform/errorMessage/common/errorMess import { ConnectionProfile } from 'sql/platform/connection/common/connectionProfile'; import { IConnectionProfile } from 'sql/platform/connection/common/interfaces'; import { escape } from 'sql/base/common/strings'; +import * as notebookUtils from 'sql/parts/notebook/notebookUtils'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; export const sqlKernel: string = localize('sqlKernel', 'SQL'); @@ -26,12 +28,23 @@ export const MAX_ROWS = 5000; export const NotebookConfigSectionName = 'notebook'; export const MaxTableRowsConfigName = 'maxTableRows'; -let sqlKernelSpec: nb.IKernelSpec = ({ +const sqlKernelSpec: nb.IKernelSpec = ({ name: sqlKernel, language: 'sql', display_name: sqlKernel }); +const languageMagics: ILanguageMagic[] = [{ + language: 'Python', + magic: 'lang_python' +}, { + language: 'R', + magic: 'lang_r' +}, { + language: 'Java', + magic: 'lang_java' +}]; + export interface SQLData { columns: Array; rows: Array>; @@ -135,12 +148,22 @@ class SqlKernel extends Disposable implements nb.IKernel { private _id: string; private _future: SQLFuture; private _executionCount: number = 0; + private _magicToExecutorMap = new Map(); - constructor( @IConnectionManagementService private _connectionManagementService: IConnectionManagementService, + constructor(@IConnectionManagementService private _connectionManagementService: IConnectionManagementService, @IInstantiationService private _instantiationService: IInstantiationService, @IErrorMessageService private _errorMessageService: IErrorMessageService, - @IConfigurationService private _configurationService: IConfigurationService) { + @IConfigurationService private _configurationService: IConfigurationService + ) { super(); + this.initMagics(); + } + + private initMagics(): void { + for (let magic of languageMagics) { + let scriptMagic = new ExternalScriptMagic(magic.language); + this._magicToExecutorMap.set(magic.magic, scriptMagic); + } } public get id(): string { @@ -197,6 +220,7 @@ class SqlKernel extends Disposable implements nb.IKernel { requestExecute(content: nb.IExecuteRequest, disposeOnDone?: boolean): nb.IFuture { let canRun: boolean = true; + let code = this.getCodeWithoutCellMagic(content); if (this._queryRunner) { // Cancel any existing query if (this._future && !this._queryRunner.hasCompleted) { @@ -204,14 +228,13 @@ class SqlKernel extends Disposable implements nb.IKernel { // TODO when we can just show error as an output, should show an "execution canceled" error in output this._future.handleDone(); } - this._queryRunner.runQuery(content.code); + this._queryRunner.runQuery(code); } else if (this._currentConnection) { let connectionUri = Utils.generateUri(this._currentConnection, 'notebook'); this._queryRunner = this._instantiationService.createInstance(QueryRunner, connectionUri); - this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) => - { + this._connectionManagementService.connect(this._currentConnection, connectionUri).then((result) => { this.addQueryEventListeners(this._queryRunner); - this._queryRunner.runQuery(content.code); + this._queryRunner.runQuery(code); }); } else { canRun = false; @@ -231,6 +254,25 @@ class SqlKernel extends Disposable implements nb.IKernel { return this._future; } + private getCodeWithoutCellMagic(content: nb.IExecuteRequest): string { + let code = content.code; + let firstLineEnd = code.indexOf(os.EOL); + let firstLine = code.substring(0, (firstLineEnd >= 0) ? firstLineEnd : 0).trimLeft(); + if (firstLine.startsWith('%%')) { + // Strip out the line + code = code.substring(firstLineEnd, code.length); + // Try and match to an external script magic. If we add more magics later, should handle transforms better + let magic = notebookUtils.tryMatchCellMagic(firstLine); + if (magic) { + let executor = this._magicToExecutorMap.get(magic.toLowerCase()); + if (executor) { + code = executor.convertToExternalScript(code); + } + } + } + return code; + } + requestComplete(content: nb.ICompleteRequest): Thenable { let response: Partial = {}; return Promise.resolve(response as nb.ICompleteReplyMsg); @@ -388,7 +430,7 @@ export class SQLFuture extends Disposable implements FutureInternal { private convertToDataResource(columns: IDbColumn[], subsetResult: QueryExecuteSubsetResult): IDataResource { let columnsResources: IDataResourceSchema[] = []; columns.forEach(column => { - columnsResources.push({name: escape(column.columnName)}); + columnsResources.push({ name: escape(column.columnName) }); }); let columnsFields: IDataResourceFields = { fields: undefined }; columnsFields.fields = columnsResources; @@ -482,4 +524,17 @@ export interface IDataResourceFields { export interface IDataResourceSchema { name: string; type?: string; -} \ No newline at end of file +} + +class ExternalScriptMagic { + + constructor(private language: string) { + } + + public convertToExternalScript(script: string): string { + return `execute sp_execute_external_script + @language = N'${this.language}', + @script = N'${script}' + `; + } +} diff --git a/src/sqltest/parts/notebook/common.ts b/src/sqltest/parts/notebook/common.ts index f10e000300..d558749deb 100644 --- a/src/sqltest/parts/notebook/common.ts +++ b/src/sqltest/parts/notebook/common.ts @@ -17,6 +17,7 @@ export class NotebookModelStub implements INotebookModel { constructor(private _languageInfo?: nb.ILanguageInfo) { } public trustedMode: boolean; + language: string; public get languageInfo(): nb.ILanguageInfo { return this._languageInfo; diff --git a/src/sqltest/parts/notebook/model/cell.test.ts b/src/sqltest/parts/notebook/model/cell.test.ts index 3da6f796b3..58c52f5550 100644 --- a/src/sqltest/parts/notebook/model/cell.test.ts +++ b/src/sqltest/parts/notebook/model/cell.test.ts @@ -27,7 +27,7 @@ describe('Cell Model', function (): void { it('Should update values', async function (): Promise { let cell = factory.createCell(undefined, undefined); - cell.language = 'sql'; + cell.setOverrideLanguage('sql'); should(cell.language).equal('sql'); cell.source = 'abcd'; should(cell.source).equal('abcd'); diff --git a/src/sqltest/parts/notebook/model/notebookModel.test.ts b/src/sqltest/parts/notebook/model/notebookModel.test.ts index 1bdf8449db..a875d4e562 100644 --- a/src/sqltest/parts/notebook/model/notebookModel.test.ts +++ b/src/sqltest/parts/notebook/model/notebookModel.test.ts @@ -96,6 +96,7 @@ describe('notebook model', function(): void { connectionService: queryConnectionService.object, providerId: 'SQL', standardKernels: [{ name: 'SQL', connectionProviderIds: ['MSSQL'], notebookProvider: 'sql' }], + cellMagicMapper: undefined, defaultKernel: undefined, layoutChanged: undefined, capabilitiesService: capabilitiesService.object